Add Goodreads series and group them on the about page

tools/content/goodreads_series.py

@@ -0,0 +1,169 @@
+import html
+import json
+import re
+
+import goodreads
+
+SERIES_URL_PATTERN = re.compile(r"goodreads\.com/series/(\d+)")
+REACT_PROPS_PATTERN = re.compile(r"data-react-props=\"([^\"]*)\"", re.S)
+BOOK_LINK_PATTERN = re.compile(r"<a[^>]+href=\"(?:https://www\.goodreads\.com)?/book/show/(\d+)[^\"]*\"[^>]*>\s*(?:<span[^>]*>)?\s*([^<]+?)\s*<", re.S)
+SERIES_TITLE_SUFFIX = re.compile(r"\s+(?:Series\s+)?by\s+.*$", re.I)
+
+
+def series_id(url: str) -> str | None:
+    match = SERIES_URL_PATTERN.search(url)
+    return match.group(1) if match else None
+
+
+def first_text(data: dict, keys: list[str]) -> str:
+    for key in keys:
+        value = data.get(key)
+        if isinstance(value, str) and value.strip():
+            return html.unescape(value.strip())
+    return ""
+
+
+def author_names(book: dict) -> list[str]:
+    author = book.get("author")
+    if isinstance(author, dict) and author.get("name"):
+        return [html.unescape(author["name"])]
+    authors = book.get("authors")
+    if isinstance(authors, list):
+        return [html.unescape(item["name"]) for item in authors if isinstance(item, dict) and item.get("name")]
+    name = first_text(book, ["authorName"])
+    return [name] if name else []
+
+
+def entry_from_book(book: dict, position: str) -> dict | None:
+    identifier = str(book.get("bookId") or book.get("legacyId") or book.get("id") or "")
+    link = first_text(book, ["bookUrl", "url", "webUrl"])
+    if not identifier.isdigit():
+        identifier = goodreads.book_id(link) or ""
+    if not identifier:
+        return None
+    return {
+        "goodreadsId": identifier,
+        "title": first_text(book, ["bookTitleBare", "title", "titleComplete"]),
+        "position": position,
+        "authors": author_names(book),
+        "cover": first_text(book, ["imageUrl", "image"]) or None,
+        "url": f"https://www.goodreads.com/book/show/{identifier}",
+    }
+
+
+def clean_header(value) -> str:
+    return html.unescape(value).strip() if isinstance(value, str) else ""
+
+
+def own_header(item: dict) -> str:
+    return next((clean_header(value) for key, value in item.items() if "header" in key.lower() and isinstance(value, str)), "")
+
+
+def is_book_list(value) -> bool:
+    return isinstance(value, list) and bool(value) and all(isinstance(item, dict) and isinstance(item.get("book"), dict) for item in value)
+
+
+def collect_entries(node, found: list[dict]):
+    if isinstance(node, list):
+        for item in node:
+            collect_entries(item, found)
+        return
+    if not isinstance(node, dict):
+        return
+
+    if isinstance(node.get("book"), dict):
+        entry = entry_from_book(node["book"], own_header(node))
+        if entry:
+            found.append(entry)
+        return
+
+    header_lists = [value for key, value in node.items() if "header" in key.lower() and isinstance(value, list)]
+    for value in node.values():
+        if not is_book_list(value):
+            collect_entries(value, found)
+            continue
+        headers = next((headers for headers in header_lists if len(headers) == len(value)), [])
+        for index, item in enumerate(value):
+            position = clean_header(headers[index]) if index < len(headers) else ""
+            entry = entry_from_book(item["book"], position or own_header(item))
+            if entry:
+                found.append(entry)
+
+
+def entries_from_react_props(page: str) -> list[dict]:
+    found: list[dict] = []
+    for raw in REACT_PROPS_PATTERN.findall(page):
+        try:
+            collect_entries(json.loads(html.unescape(raw)), found)
+        except ValueError:
+            continue
+    return found
+
+
+def entries_from_next_data(page: str) -> list[dict]:
+    state = goodreads.apollo_state(page)
+    books = [value for value in state.values() if isinstance(value, dict) and value.get("__typename") == "Book"]
+    entries = []
+    for book in books:
+        entry = entry_from_book(book, "")
+        if entry:
+            entry["authors"] = goodreads.book_authors(state, book) or entry["authors"]
+            entries.append(entry)
+    return entries
+
+
+def entries_from_links(page: str) -> list[dict]:
+    entries = []
+    for identifier, title in BOOK_LINK_PATTERN.findall(page):
+        title = html.unescape(title).strip()
+        if title:
+            entries.append({"goodreadsId": identifier, "title": title, "position": "", "authors": [], "cover": None, "url": f"https://www.goodreads.com/book/show/{identifier}"})
+    return entries
+
+
+def unique_entries(entries: list[dict]) -> list[dict]:
+    seen = set()
+    unique = []
+    for entry in entries:
+        if entry["goodreadsId"] in seen:
+            continue
+        seen.add(entry["goodreadsId"])
+        unique.append(entry)
+    return unique
+
+
+def series_title(page: str) -> str:
+    meta = goodreads.meta_tags(page)
+    title = meta.get("og:title") or goodreads.page_title(page)
+    return SERIES_TITLE_SUFFIX.sub("", title).strip()
+
+
+def scrape_series(url: str) -> dict:
+    identifier = series_id(url)
+    if not identifier:
+        raise ValueError(f"not a Goodreads series url: {url}")
+    page = goodreads.download_page(url)
+    reason = goodreads.blocked_reason(page.text)
+    if reason:
+        raise ValueError(reason)
+
+    entries = entries_from_react_props(page.text) or entries_from_next_data(page.text) or entries_from_links(page.text)
+    entries = unique_entries(entries)
+    if not entries:
+        raise ValueError("Could not find any books on the series page")
+
+    return {
+        "seriesId": identifier,
+        "name": series_title(page.text),
+        "url": f"https://www.goodreads.com/series/{identifier}",
+        "books": entries,
+    }
+
+
+def print_series(series: dict):
+    print(f"{series['name']}  ({len(series['books'])} books)")
+    print("-" * 60)
+    for entry in series["books"]:
+        position = entry["position"] or "-"
+        authors = ", ".join(entry["authors"]) or "unknown author"
+        print(f"{position:<12} {entry['title']}  by {authors}  [{entry['goodreadsId']}]")