Add SQLite library with API, library desktop apps and MobyGames scraper

tools/content/goodreads.py

@@ -4,51 +4,25 @@ import html
 import json
 import re
 import sys
-import urllib.request
 from pathlib import Path
 
+from html_page import PageResult, download_page, meta_tags, page_title, plain_text
+
 BOOK_URL_PATTERN = re.compile(r"goodreads\.com/(?:\w+/)?book/show/(\d+)")
 JSON_LD_PATTERN = re.compile(r"<script[^>]*type=[\"']application/ld\+json[\"'][^>]*>(.*?)</script>", re.S | re.I)
 NEXT_DATA_PATTERN = re.compile(r"<script[^>]*id=[\"']__NEXT_DATA__[\"'][^>]*>(.*?)</script>", re.S | re.I)
-META_TAG_PATTERN = re.compile(r"<meta\s[^>]*>", re.S | re.I)
-ATTRIBUTE_PATTERN = re.compile(r"([\w:-]+)=\"([^\"]*)\"|([\w:-]+)='([^']*)'")
-PAGE_TITLE_PATTERN = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
-TAG_PATTERN = re.compile(r"<[^>]+>")
 BLOCKED_PAGE_HINTS = ["captcha", "awswaf", "robot check", "access denied", "are you a human", "unusual traffic"]
-REQUEST_HEADERS = {
-    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
-    "Accept": "text/html,application/xhtml+xml",
-    "Accept-Language": "en-US,en;q=0.9",
-}
 
 
 class BlockedError(ValueError):
     pass
 
 
-class PageResult:
-    def __init__(self, status: int, final_url: str, text: str):
-        self.status = status
-        self.final_url = final_url
-        self.text = text
-
-
 def book_id(url: str) -> str | None:
     match = BOOK_URL_PATTERN.search(url)
     return match.group(1) if match else None
 
 
-def download_page(url: str) -> PageResult:
-    request = urllib.request.Request(url, headers=REQUEST_HEADERS)
-    with urllib.request.urlopen(request, timeout=15) as response:
-        return PageResult(response.status, response.geturl(), response.read().decode("utf-8", errors="replace"))
-
-
-def page_title(page: str) -> str:
-    match = PAGE_TITLE_PATTERN.search(page)
-    return html.unescape(match.group(1)).strip() if match else ""
-
-
 def blocked_reason(page: str) -> str | None:
     lowered = page.lower()
     hint = next((hint for hint in BLOCKED_PAGE_HINTS if hint in lowered), None)
@@ -71,25 +45,6 @@ def json_ld_book(page: str) -> dict:
     return {}
 
 
-def meta_tags(page: str) -> dict:
-    tags = {}
-    for tag in META_TAG_PATTERN.findall(page):
-        attributes = {}
-        for double_name, double_value, single_name, single_value in ATTRIBUTE_PATTERN.findall(tag):
-            attributes[(double_name or single_name).lower()] = double_value or single_value
-        name = attributes.get("property") or attributes.get("name")
-        if name and "content" in attributes:
-            tags[name] = html.unescape(attributes["content"])
-    return tags
-
-
-def plain_text(text: str | None) -> str | None:
-    if not text:
-        return None
-    with_breaks = re.sub(r"<br\s*/?>", "\n", text)
-    return html.unescape(TAG_PATTERN.sub("", with_breaks)).strip() or None
-
-
 def apollo_state(page: str) -> dict:
     match = NEXT_DATA_PATTERN.search(page)
     if not match: