Cache Goodreads data and slow down bulk requests

.gitignore

@@ -32,4 +32,5 @@ blogs/
 
 # Content tool
 tools/content/.venv/
+tools/content/.cache/
 __pycache__/

tools/content/book_cache.py

@@ -0,0 +1,75 @@
+import json
+import re
+import urllib.request
+
+import goodreads
+from jsonfile import PROJECT_ROOT
+
+SCRAPE_CACHE_DIR = PROJECT_ROOT / "tools" / "content" / ".cache" / "goodreads"
+COVERS_DIR = PROJECT_ROOT / "static" / "covers" / "books"
+COVERS_PUBLIC_PATH = "/covers/books"
+IMAGE_EXTENSION_PATTERN = re.compile(r"\.(jpe?g|png|webp|gif)(?:$|\?)", re.I)
+
+
+def scrape_cache_path(identifier: str):
+    return SCRAPE_CACHE_DIR / f"{identifier}.json"
+
+
+def cached_scrape(identifier: str) -> dict | None:
+    path = scrape_cache_path(identifier)
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except ValueError:
+        return None
+
+
+def store_scrape(scraped: dict) -> None:
+    SCRAPE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    scrape_cache_path(scraped["goodreadsId"]).write_text(json.dumps(scraped, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def scrape_book(url: str, fresh: bool = False) -> dict:
+    identifier = goodreads.book_id(url)
+    cached = None if fresh or not identifier else cached_scrape(identifier)
+    if cached:
+        return cached
+    scraped = goodreads.scrape_book(url)
+    if scraped.get("title"):
+        store_scrape(scraped)
+    return scraped
+
+
+def is_local_cover(cover: str | None) -> bool:
+    return bool(cover) and cover.startswith(COVERS_PUBLIC_PATH)
+
+
+def existing_cover(identifier: str):
+    return next(iter(sorted(COVERS_DIR.glob(f"{identifier}.*"))), None) if COVERS_DIR.exists() else None
+
+
+def cache_cover(identifier: str, source_url: str | None, fresh: bool = False) -> str | None:
+    existing = existing_cover(identifier)
+    if existing and not fresh:
+        return f"{COVERS_PUBLIC_PATH}/{existing.name}"
+    if not source_url or is_local_cover(source_url):
+        return f"{COVERS_PUBLIC_PATH}/{existing.name}" if existing else None
+
+    match = IMAGE_EXTENSION_PATTERN.search(source_url)
+    extension = match.group(1).lower().replace("jpeg", "jpg") if match else "jpg"
+    request = urllib.request.Request(source_url, headers=goodreads.REQUEST_HEADERS)
+    with urllib.request.urlopen(request, timeout=15) as response:
+        image = response.read()
+
+    COVERS_DIR.mkdir(parents=True, exist_ok=True)
+    if existing:
+        existing.unlink()
+    (COVERS_DIR / f"{identifier}.{extension}").write_bytes(image)
+    return f"{COVERS_PUBLIC_PATH}/{identifier}.{extension}"
+
+
+def remove_cover(identifier: str) -> None:
+    existing = existing_cover(identifier)
+    if existing:
+        existing.unlink()

tools/content/books.py

@@ -1,5 +1,6 @@
 import datetime
 
+import book_cache
 import goodreads
 import jsonfile
 
@@ -47,13 +48,21 @@ def short_description(text: str | None) -> str | None:
     return text[:DESCRIPTION_MAX_LENGTH].rsplit(" ", 1)[0] + "..."
 
 
+def local_cover(scraped: dict) -> str | None:
+    try:
+        return book_cache.cache_cover(scraped["goodreadsId"], scraped["cover"]) or scraped["cover"]
+    except OSError:
+        return scraped["cover"]
+
+
 def stored_fields(scraped: dict) -> dict:
     return {
         "goodreadsId": scraped["goodreadsId"],
         "url": scraped["url"],
         "title": scraped["title"],
         "authors": scraped["authors"],
-        "cover": scraped["cover"],
+        "cover": local_cover(scraped),
+        "coverSource": scraped["cover"],
         "description": short_description(scraped["description"]),
         "pages": scraped["pages"],
         "published": scraped["published"],
@@ -107,7 +116,7 @@ def add_book(url: str, shelf: str = "read", rating: int | None = None) -> dict:
     if find_index(load_books(), identifier) is not None:
         raise ValueError(f"{identifier} is already listed")
 
-    scraped = goodreads.scrape_book(url)
+    scraped = book_cache.scrape_book(url)
     if not scraped["title"]:
         raise ValueError("Could not read the book from Goodreads")
 
@@ -130,6 +139,7 @@ def remove_book(text: str) -> dict:
         raise ValueError(f"{text} is not listed")
     removed = books.pop(index)
     save_books(books)
+    book_cache.remove_cover(removed["goodreadsId"])
     return removed
 
 
@@ -156,5 +166,18 @@ def refresh_book(text: str) -> dict:
     index = find_index(books, text)
     if index is None:
         raise ValueError(f"{text} is not listed")
-    scraped = goodreads.scrape_book(books[index]["url"])
+    scraped = book_cache.scrape_book(books[index]["url"], fresh=True)
     return update_book(text, lambda book: {**book, **stored_fields(scraped)})
+
+
+def books_missing_local_covers() -> list[dict]:
+    return [book for book in load_books() if not book_cache.is_local_cover(book.get("cover"))]
+
+
+def cache_book_cover(identifier: str) -> dict:
+    def change(book: dict) -> dict:
+        source = book.get("coverSource") or book.get("cover")
+        cover = book_cache.cache_cover(identifier, source)
+        return {**book, "cover": cover or book.get("cover"), "coverSource": source}
+
+    return update_book(identifier, change)

tools/content/goodreads.py

@@ -22,6 +22,10 @@ REQUEST_HEADERS = {
 }
 
 
+class BlockedError(ValueError):
+    pass
+
+
 class PageResult:
     def __init__(self, status: int, final_url: str, text: str):
         self.status = status
@@ -168,10 +172,14 @@ def scrape_book(url: str) -> dict:
     if not identifier:
         raise ValueError(f"not a Goodreads book url: {url}")
     page = download_page(url)
+    raise_if_blocked(page)
+    return parse_book(page.text, identifier)
+
+
+def raise_if_blocked(page: PageResult):
     reason = blocked_reason(page.text)
     if reason:
-        raise ValueError(reason)
+        raise BlockedError(f"Goodreads is rate limiting requests ({reason}), wait a few minutes and try again")
-    return parse_book(page.text, identifier)
 
 
 def print_diagnostics(url: str, save_path: Path | None):

tools/content/goodreads_series.py

@@ -143,9 +143,7 @@ def scrape_series(url: str) -> dict:
     if not identifier:
         raise ValueError(f"not a Goodreads series url: {url}")
     page = goodreads.download_page(url)
-    reason = goodreads.blocked_reason(page.text)
+    goodreads.raise_if_blocked(page)
-    if reason:
-        raise ValueError(reason)
 
     entries = entries_from_react_props(page.text) or entries_from_next_data(page.text) or entries_from_links(page.text)
     entries = unique_entries(entries)

tools/content/tui/books_pane.py

@@ -1,3 +1,5 @@
+import time
+
 from rich.markup import escape
 from textual import on, work
 from textual.app import ComposeResult
@@ -6,11 +8,14 @@ from textual.containers import Horizontal, VerticalScroll
 from textual.widgets import DataTable, Static
 
 import books
+import goodreads
 import goodreads_series
 from tui.content_pane import ContentPane
 from tui.dialogs import ConfirmDialog, FormDialog, FormField
 from tui.series_picker import SeriesPicker
 
+REQUEST_DELAY_SECONDS = 3
+
 
 def book_details(book: dict) -> str:
     rows = [
@@ -40,6 +45,7 @@ class BooksPane(ContentPane):
         Binding("d", "delete", "Delete"),
         Binding("u", "refresh", "Refresh"),
         Binding("U", "refresh_all", "Refresh all"),
+        Binding("C", "cache_covers", "Cache covers"),
         Binding("r", "reload", "Reload"),
     ]
 
@@ -151,19 +157,48 @@ class BooksPane(ContentPane):
         self.notify("Refreshing every book from Goodreads...")
         self.refresh_every_book()
 
+    def report_blocked(self, error: Exception):
+        self.app.call_from_thread(self.notify, str(error), title="Stopped", severity="error", timeout=15)
+
     @work(thread=True, exclusive=True, group="refresh")
     def refresh_every_book(self):
-        failures = 0
+        refreshed = 0
         all_books = books.load_books()
-        for book in all_books:
+        for index, book in enumerate(all_books):
+            if index:
+                time.sleep(REQUEST_DELAY_SECONDS)
             try:
                 books.refresh_book(book["goodreadsId"])
+            except goodreads.BlockedError as error:
+                self.report_blocked(error)
+                break
             except (OSError, ValueError) as error:
-                failures += 1
                 self.app.call_from_thread(self.notify, f"{book['title']}: {error}", title="Skipped", severity="warning")
+                continue
+            refreshed += 1
         self.app.call_from_thread(lambda: self.reload(self.selected_key()))
-        summary = f"Refreshed {len(all_books) - failures} of {len(all_books)} books"
+        self.app.call_from_thread(self.notify, f"Refreshed {refreshed} of {len(all_books)} books")
-        self.app.call_from_thread(self.notify, summary)
+
+    def action_cache_covers(self):
+        missing = books.books_missing_local_covers()
+        if not missing:
+            self.notify("Every cover is already saved in static/covers/books")
+            return
+        self.notify(f"Saving {len(missing)} covers...")
+        self.cache_missing_covers(missing)
+
+    @work(thread=True, exclusive=True, group="covers")
+    def cache_missing_covers(self, missing: list[dict]):
+        saved = 0
+        for book in missing:
+            try:
+                books.cache_book_cover(book["goodreadsId"])
+            except (OSError, ValueError) as error:
+                self.app.call_from_thread(self.notify, f"{book['title']}: {error}", title="Skipped", severity="warning")
+                continue
+            saved += 1
+        self.app.call_from_thread(lambda: self.reload(self.selected_key()))
+        self.app.call_from_thread(self.notify, f"Saved {saved} of {len(missing)} covers")
 
     def action_add_series(self):
         fields = [FormField("link", "Goodreads series link", placeholder="https://www.goodreads.com/series/92037-wings-of-fire", required=True)]
@@ -185,12 +220,21 @@ class BooksPane(ContentPane):
     @work(thread=True, exclusive=True, group="series")
     def add_series_books(self, series: dict, choices: dict[str, str]):
         added = 0
+        already_listed = set(books.library_shelves())
+        fetched_any = False
         for entry in series["books"]:
             shelf = choices.get(entry["goodreadsId"])
             if not shelf:
                 continue
+            needs_fetch = entry["goodreadsId"] not in already_listed
+            if needs_fetch and fetched_any:
+                time.sleep(REQUEST_DELAY_SECONDS)
+            fetched_any = fetched_any or needs_fetch
             try:
                 book = books.add_series_book(series, entry, shelf)
+            except goodreads.BlockedError as error:
+                self.report_blocked(error)
+                break
             except (OSError, ValueError) as error:
                 self.app.call_from_thread(self.notify, f"{entry['title']}: {error}", title="Skipped", severity="warning")
                 continue