Add SQLite library with API, library desktop apps and MobyGames scraper

tools/content/html_page.py

@@ -0,0 +1,51 @@
+import html
+import re
+import urllib.request
+
+META_TAG_PATTERN = re.compile(r"<meta\s[^>]*>", re.S | re.I)
+ATTRIBUTE_PATTERN = re.compile(r"([\w:-]+)=\"([^\"]*)\"|([\w:-]+)='([^']*)'")
+PAGE_TITLE_PATTERN = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
+TAG_PATTERN = re.compile(r"<[^>]+>")
+
+REQUEST_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    "Accept": "text/html,application/xhtml+xml",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+
+class PageResult:
+    def __init__(self, status: int, final_url: str, text: str):
+        self.status = status
+        self.final_url = final_url
+        self.text = text
+
+
+def download_page(url: str) -> PageResult:
+    request = urllib.request.Request(url, headers=REQUEST_HEADERS)
+    with urllib.request.urlopen(request, timeout=15) as response:
+        return PageResult(response.status, response.geturl(), response.read().decode("utf-8", errors="replace"))
+
+
+def page_title(page: str) -> str:
+    match = PAGE_TITLE_PATTERN.search(page)
+    return html.unescape(match.group(1)).strip() if match else ""
+
+
+def meta_tags(page: str) -> dict:
+    tags = {}
+    for tag in META_TAG_PATTERN.findall(page):
+        attributes = {}
+        for double_name, double_value, single_name, single_value in ATTRIBUTE_PATTERN.findall(tag):
+            attributes[(double_name or single_name).lower()] = double_value or single_value
+        name = attributes.get("property") or attributes.get("name")
+        if name and "content" in attributes:
+            tags[name] = html.unescape(attributes["content"])
+    return tags
+
+
+def plain_text(text: str | None) -> str | None:
+    if not text:
+        return None
+    with_breaks = re.sub(r"<br\s*/?>", "\n", text)
+    return html.unescape(TAG_PATTERN.sub("", with_breaks)).strip() or None