classes.c30_scraper

  1import wikipedia
  2import re
  3from loguru import logger
  4
  5from lib import helpers, content
  6
  7
  8class KScraper:
  9    def __init__(self, folder="01 Content/articles", language="en") -> None:
 10        """
 11        Initializes the KScraper with a target folder and language.
 12
 13        Args:
 14            folder: Folder partial where articles will be saved.
 15            language: The language for Wikipedia articles.
 16        """
 17        # Prepend absolute path in case specimen not run from 03 DrawBot/
 18        self.folder = helpers.createFolder(
 19            f"/Users/christianjansky/Library/CloudStorage/Dropbox/KOMETA-Work/40 Scripts/03 DrawBot/{folder.rstrip('/')}-{language}/"
 20        )
 21        """Full path to the file system folder where articles will be saved."""
 22        wikipedia.set_lang(language)
 23
 24    def fetchArticle(self, slug: str) -> str:
 25        """
 26        Fetches a Wikipedia article by slug, saves it, and returns parsed content.
 27
 28        Args:
 29            slug: The Wikipedia page slug.
 30
 31        Returns:
 32            The parsed article content.
 33        """
 34
 35        def _findWikiTitle(slug: str) -> str:
 36            """
 37            Finds a valid Wikipedia page title or its closest match.
 38
 39            Args:
 40                slug: The Wikipedia page slug.
 41
 42            Returns:
 43                The resolved Wikipedia page title.
 44            """
 45            try:
 46                try:
 47                    # Find exact match
 48                    page = wikipedia.page(slug, auto_suggest=False)
 49                except wikipedia.exceptions.PageError as e:
 50                    # Correct spelling error
 51                    suggested = wikipedia.suggest(e.pageid)
 52                    logger.warning("Correcting {} to {}", e.pageid, suggested)
 53                    page = wikipedia.page(suggested, auto_suggest=False)
 54            except wikipedia.exceptions.DisambiguationError as e:
 55                # Pick first possible match if it may refer to multiple pages
 56                logger.warning("Possible pages for {}: {}", slug, e.options)
 57                page = wikipedia.page(helpers.pickFirst(e.options), auto_suggest=False)
 58
 59            return page.title
 60
 61        def _fixPageContent(input: str) -> str:
 62            """
 63            Fixes collapsed spacing in Wikipedia content.
 64
 65            Args:
 66                input: The raw Wikipedia content.
 67
 68            Returns:
 69                The fixed content with proper spacing.
 70            """
 71            output = re.sub(r"([a-z0-9]\.)([A-Z])", r"\1 \2", input)
 72            return output
 73
 74        def _saveArticle(slug: str) -> str:
 75            """
 76            Saves the Wikipedia article as a .txt file and returns its content.
 77
 78            Args:
 79                slug: The Wikipedia page slug.
 80
 81            Returns:
 82                The saved article content.
 83            """
 84            logger.debug("Saving article for {}", slug)
 85            with open(locateFile(slug), "w") as file:
 86                page = wikipedia.page(slug, auto_suggest=False)
 87                output = f"{page.url}\n\n{_fixPageContent(page.content)}"
 88                file.write(output)
 89                return output
 90
 91        def _parseArticle(input: str) -> str:
 92            """
 93            Parses the saved article content, removing URL and formatting.
 94
 95            Args:
 96                input: The raw article content.
 97
 98            Returns:
 99                The cleaned article content.
100            """
101            # Strip article URL from first line
102            output = re.sub(r"^http.+\n+", "", input)
103            # Strip == Headline == formatting
104            output = re.sub(r"(?<=\n)=+\s+|\s+=+(?=\n)", "", output)
105
106            return output
107
108        pageTitle = _findWikiTitle(slug)
109        locateFile = (
110            lambda pageSlug: f"{self.folder}wiki-{content.toKebabCase(pageSlug)}.txt"
111        )
112        filePath = locateFile(pageTitle)
113
114        if helpers.isFile(filePath):
115            result = helpers.readFile(filePath)
116        else:
117            try:
118                result = _saveArticle(pageTitle)
119            except Exception as e:
120                logger.warning("Unable to save {}: {}", pageTitle, e)
121
122        return _parseArticle(result)
123
124    def fetchArticles(self, slugs: list[str]) -> str:
125        """
126        Fetches and parses multiple Wikipedia articles.
127
128        Args:
129            slugs: A list of Wikipedia page slugs.
130
131        Returns:
132            The concatenated content of all fetched articles.
133        """
134        return "\n".join(map(self.fetchArticle, slugs))