website/tools/content/goodreads_series.py

import html
import json
import re

import goodreads

SERIES_URL_PATTERN = re.compile(r"goodreads\.com/series/(\d+)")
REACT_PROPS_PATTERN = re.compile(r"data-react-props=\"([^\"]*)\"", re.S)
BOOK_LINK_PATTERN = re.compile(r"<a[^>]+href=\"(?:https://www\.goodreads\.com)?/book/show/(\d+)[^\"]*\"[^>]*>\s*(?:<span[^>]*>)?\s*([^<]+?)\s*<", re.S)
SERIES_TITLE_SUFFIX = re.compile(r"\s+(?:Series\s+)?by\s+.*$", re.I)


def series_id(url: str) -> str | None:
    match = SERIES_URL_PATTERN.search(url)
    return match.group(1) if match else None


def first_text(data: dict, keys: list[str]) -> str:
    for key in keys:
        value = data.get(key)
        if isinstance(value, str) and value.strip():
            return html.unescape(value.strip())
    return ""


def author_names(book: dict) -> list[str]:
    author = book.get("author")
    if isinstance(author, dict) and author.get("name"):
        return [html.unescape(author["name"])]
    authors = book.get("authors")
    if isinstance(authors, list):
        return [html.unescape(item["name"]) for item in authors if isinstance(item, dict) and item.get("name")]
    name = first_text(book, ["authorName"])
    return [name] if name else []


def entry_from_book(book: dict, position: str) -> dict | None:
    identifier = str(book.get("bookId") or book.get("legacyId") or book.get("id") or "")
    link = first_text(book, ["bookUrl", "url", "webUrl"])
    if not identifier.isdigit():
        identifier = goodreads.book_id(link) or ""
    if not identifier:
        return None
    return {
        "goodreadsId": identifier,
        "title": first_text(book, ["bookTitleBare", "title", "titleComplete"]),
        "position": position,
        "authors": author_names(book),
        "cover": first_text(book, ["imageUrl", "image"]) or None,
        "url": f"https://www.goodreads.com/book/show/{identifier}",
    }


def clean_header(value) -> str:
    return html.unescape(value).strip() if isinstance(value, str) else ""


def own_header(item: dict) -> str:
    return next((clean_header(value) for key, value in item.items() if "header" in key.lower() and isinstance(value, str)), "")


def is_book_list(value) -> bool:
    return isinstance(value, list) and bool(value) and all(isinstance(item, dict) and isinstance(item.get("book"), dict) for item in value)


def collect_entries(node, found: list[dict]):
    if isinstance(node, list):
        for item in node:
            collect_entries(item, found)
        return
    if not isinstance(node, dict):
        return

    if isinstance(node.get("book"), dict):
        entry = entry_from_book(node["book"], own_header(node))
        if entry:
            found.append(entry)
        return

    header_lists = [value for key, value in node.items() if "header" in key.lower() and isinstance(value, list)]
    for value in node.values():
        if not is_book_list(value):
            collect_entries(value, found)
            continue
        headers = next((headers for headers in header_lists if len(headers) == len(value)), [])
        for index, item in enumerate(value):
            position = clean_header(headers[index]) if index < len(headers) else ""
            entry = entry_from_book(item["book"], position or own_header(item))
            if entry:
                found.append(entry)


def entries_from_react_props(page: str) -> list[dict]:
    found: list[dict] = []
    for raw in REACT_PROPS_PATTERN.findall(page):
        try:
            collect_entries(json.loads(html.unescape(raw)), found)
        except ValueError:
            continue
    return found


def entries_from_next_data(page: str) -> list[dict]:
    state = goodreads.apollo_state(page)
    books = [value for value in state.values() if isinstance(value, dict) and value.get("__typename") == "Book"]
    entries = []
    for book in books:
        entry = entry_from_book(book, "")
        if entry:
            entry["authors"] = goodreads.book_authors(state, book) or entry["authors"]
            entries.append(entry)
    return entries


def entries_from_links(page: str) -> list[dict]:
    entries = []
    for identifier, title in BOOK_LINK_PATTERN.findall(page):
        title = html.unescape(title).strip()
        if title:
            entries.append({"goodreadsId": identifier, "title": title, "position": "", "authors": [], "cover": None, "url": f"https://www.goodreads.com/book/show/{identifier}"})
    return entries


def unique_entries(entries: list[dict]) -> list[dict]:
    seen = set()
    unique = []
    for entry in entries:
        if entry["goodreadsId"] in seen:
            continue
        seen.add(entry["goodreadsId"])
        unique.append(entry)
    return unique


def series_title(page: str) -> str:
    meta = goodreads.meta_tags(page)
    title = meta.get("og:title") or goodreads.page_title(page)
    return SERIES_TITLE_SUFFIX.sub("", title).strip()


def scrape_series(url: str) -> dict:
    identifier = series_id(url)
    if not identifier:
        raise ValueError(f"not a Goodreads series url: {url}")
    page = goodreads.download_page(url)
    goodreads.raise_if_blocked(page)

    entries = entries_from_react_props(page.text) or entries_from_next_data(page.text) or entries_from_links(page.text)
    entries = unique_entries(entries)
    if not entries:
        raise ValueError("Could not find any books on the series page")

    return {
        "seriesId": identifier,
        "name": series_title(page.text),
        "url": f"https://www.goodreads.com/series/{identifier}",
        "books": entries,
    }


def print_series(series: dict):
    print(f"{series['name']}  ({len(series['books'])} books)")
    print("-" * 60)
    for entry in series["books"]:
        position = entry["position"] or "-"
        authors = ", ".join(entry["authors"]) or "unknown author"
        print(f"{position:<12} {entry['title']}  by {authors}  [{entry['goodreadsId']}]")