diff --git a/.gitignore b/.gitignore index eda77b5..eef90cf 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ blogs/ # Content tool tools/content/.venv/ +tools/content/.cache/ __pycache__/ diff --git a/tools/content/book_cache.py b/tools/content/book_cache.py new file mode 100644 index 0000000..8120757 --- /dev/null +++ b/tools/content/book_cache.py @@ -0,0 +1,75 @@ +import json +import re +import urllib.request + +import goodreads +from jsonfile import PROJECT_ROOT + +SCRAPE_CACHE_DIR = PROJECT_ROOT / "tools" / "content" / ".cache" / "goodreads" +COVERS_DIR = PROJECT_ROOT / "static" / "covers" / "books" +COVERS_PUBLIC_PATH = "/covers/books" +IMAGE_EXTENSION_PATTERN = re.compile(r"\.(jpe?g|png|webp|gif)(?:$|\?)", re.I) + + +def scrape_cache_path(identifier: str): + return SCRAPE_CACHE_DIR / f"{identifier}.json" + + +def cached_scrape(identifier: str) -> dict | None: + path = scrape_cache_path(identifier) + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except ValueError: + return None + + +def store_scrape(scraped: dict) -> None: + SCRAPE_CACHE_DIR.mkdir(parents=True, exist_ok=True) + scrape_cache_path(scraped["goodreadsId"]).write_text(json.dumps(scraped, ensure_ascii=False, indent=2), encoding="utf-8") + + +def scrape_book(url: str, fresh: bool = False) -> dict: + identifier = goodreads.book_id(url) + cached = None if fresh or not identifier else cached_scrape(identifier) + if cached: + return cached + scraped = goodreads.scrape_book(url) + if scraped.get("title"): + store_scrape(scraped) + return scraped + + +def is_local_cover(cover: str | None) -> bool: + return bool(cover) and cover.startswith(COVERS_PUBLIC_PATH) + + +def existing_cover(identifier: str): + return next(iter(sorted(COVERS_DIR.glob(f"{identifier}.*"))), None) if COVERS_DIR.exists() else None + + +def cache_cover(identifier: str, source_url: str | None, fresh: bool = False) -> str | None: + existing = existing_cover(identifier) + if existing and not fresh: + return f"{COVERS_PUBLIC_PATH}/{existing.name}" + if not source_url or is_local_cover(source_url): + return f"{COVERS_PUBLIC_PATH}/{existing.name}" if existing else None + + match = IMAGE_EXTENSION_PATTERN.search(source_url) + extension = match.group(1).lower().replace("jpeg", "jpg") if match else "jpg" + request = urllib.request.Request(source_url, headers=goodreads.REQUEST_HEADERS) + with urllib.request.urlopen(request, timeout=15) as response: + image = response.read() + + COVERS_DIR.mkdir(parents=True, exist_ok=True) + if existing: + existing.unlink() + (COVERS_DIR / f"{identifier}.{extension}").write_bytes(image) + return f"{COVERS_PUBLIC_PATH}/{identifier}.{extension}" + + +def remove_cover(identifier: str) -> None: + existing = existing_cover(identifier) + if existing: + existing.unlink() diff --git a/tools/content/books.py b/tools/content/books.py index 5cefae4..6978584 100644 --- a/tools/content/books.py +++ b/tools/content/books.py @@ -1,5 +1,6 @@ import datetime +import book_cache import goodreads import jsonfile @@ -47,13 +48,21 @@ def short_description(text: str | None) -> str | None: return text[:DESCRIPTION_MAX_LENGTH].rsplit(" ", 1)[0] + "..." +def local_cover(scraped: dict) -> str | None: + try: + return book_cache.cache_cover(scraped["goodreadsId"], scraped["cover"]) or scraped["cover"] + except OSError: + return scraped["cover"] + + def stored_fields(scraped: dict) -> dict: return { "goodreadsId": scraped["goodreadsId"], "url": scraped["url"], "title": scraped["title"], "authors": scraped["authors"], - "cover": scraped["cover"], + "cover": local_cover(scraped), + "coverSource": scraped["cover"], "description": short_description(scraped["description"]), "pages": scraped["pages"], "published": scraped["published"], @@ -107,7 +116,7 @@ def add_book(url: str, shelf: str = "read", rating: int | None = None) -> dict: if find_index(load_books(), identifier) is not None: raise ValueError(f"{identifier} is already listed") - scraped = goodreads.scrape_book(url) + scraped = book_cache.scrape_book(url) if not scraped["title"]: raise ValueError("Could not read the book from Goodreads") @@ -130,6 +139,7 @@ def remove_book(text: str) -> dict: raise ValueError(f"{text} is not listed") removed = books.pop(index) save_books(books) + book_cache.remove_cover(removed["goodreadsId"]) return removed @@ -156,5 +166,18 @@ def refresh_book(text: str) -> dict: index = find_index(books, text) if index is None: raise ValueError(f"{text} is not listed") - scraped = goodreads.scrape_book(books[index]["url"]) + scraped = book_cache.scrape_book(books[index]["url"], fresh=True) return update_book(text, lambda book: {**book, **stored_fields(scraped)}) + + +def books_missing_local_covers() -> list[dict]: + return [book for book in load_books() if not book_cache.is_local_cover(book.get("cover"))] + + +def cache_book_cover(identifier: str) -> dict: + def change(book: dict) -> dict: + source = book.get("coverSource") or book.get("cover") + cover = book_cache.cache_cover(identifier, source) + return {**book, "cover": cover or book.get("cover"), "coverSource": source} + + return update_book(identifier, change) diff --git a/tools/content/goodreads.py b/tools/content/goodreads.py index d17fdb2..883dfa2 100644 --- a/tools/content/goodreads.py +++ b/tools/content/goodreads.py @@ -22,6 +22,10 @@ REQUEST_HEADERS = { } +class BlockedError(ValueError): + pass + + class PageResult: def __init__(self, status: int, final_url: str, text: str): self.status = status @@ -168,10 +172,14 @@ def scrape_book(url: str) -> dict: if not identifier: raise ValueError(f"not a Goodreads book url: {url}") page = download_page(url) + raise_if_blocked(page) + return parse_book(page.text, identifier) + + +def raise_if_blocked(page: PageResult): reason = blocked_reason(page.text) if reason: - raise ValueError(reason) - return parse_book(page.text, identifier) + raise BlockedError(f"Goodreads is rate limiting requests ({reason}), wait a few minutes and try again") def print_diagnostics(url: str, save_path: Path | None): diff --git a/tools/content/goodreads_series.py b/tools/content/goodreads_series.py index aa6f6c0..b9b4c14 100644 --- a/tools/content/goodreads_series.py +++ b/tools/content/goodreads_series.py @@ -143,9 +143,7 @@ def scrape_series(url: str) -> dict: if not identifier: raise ValueError(f"not a Goodreads series url: {url}") page = goodreads.download_page(url) - reason = goodreads.blocked_reason(page.text) - if reason: - raise ValueError(reason) + goodreads.raise_if_blocked(page) entries = entries_from_react_props(page.text) or entries_from_next_data(page.text) or entries_from_links(page.text) entries = unique_entries(entries) diff --git a/tools/content/tui/books_pane.py b/tools/content/tui/books_pane.py index 6fd7c9b..d26d386 100644 --- a/tools/content/tui/books_pane.py +++ b/tools/content/tui/books_pane.py @@ -1,3 +1,5 @@ +import time + from rich.markup import escape from textual import on, work from textual.app import ComposeResult @@ -6,11 +8,14 @@ from textual.containers import Horizontal, VerticalScroll from textual.widgets import DataTable, Static import books +import goodreads import goodreads_series from tui.content_pane import ContentPane from tui.dialogs import ConfirmDialog, FormDialog, FormField from tui.series_picker import SeriesPicker +REQUEST_DELAY_SECONDS = 3 + def book_details(book: dict) -> str: rows = [ @@ -40,6 +45,7 @@ class BooksPane(ContentPane): Binding("d", "delete", "Delete"), Binding("u", "refresh", "Refresh"), Binding("U", "refresh_all", "Refresh all"), + Binding("C", "cache_covers", "Cache covers"), Binding("r", "reload", "Reload"), ] @@ -151,19 +157,48 @@ class BooksPane(ContentPane): self.notify("Refreshing every book from Goodreads...") self.refresh_every_book() + def report_blocked(self, error: Exception): + self.app.call_from_thread(self.notify, str(error), title="Stopped", severity="error", timeout=15) + @work(thread=True, exclusive=True, group="refresh") def refresh_every_book(self): - failures = 0 + refreshed = 0 all_books = books.load_books() - for book in all_books: + for index, book in enumerate(all_books): + if index: + time.sleep(REQUEST_DELAY_SECONDS) try: books.refresh_book(book["goodreadsId"]) + except goodreads.BlockedError as error: + self.report_blocked(error) + break except (OSError, ValueError) as error: - failures += 1 self.app.call_from_thread(self.notify, f"{book['title']}: {error}", title="Skipped", severity="warning") + continue + refreshed += 1 self.app.call_from_thread(lambda: self.reload(self.selected_key())) - summary = f"Refreshed {len(all_books) - failures} of {len(all_books)} books" - self.app.call_from_thread(self.notify, summary) + self.app.call_from_thread(self.notify, f"Refreshed {refreshed} of {len(all_books)} books") + + def action_cache_covers(self): + missing = books.books_missing_local_covers() + if not missing: + self.notify("Every cover is already saved in static/covers/books") + return + self.notify(f"Saving {len(missing)} covers...") + self.cache_missing_covers(missing) + + @work(thread=True, exclusive=True, group="covers") + def cache_missing_covers(self, missing: list[dict]): + saved = 0 + for book in missing: + try: + books.cache_book_cover(book["goodreadsId"]) + except (OSError, ValueError) as error: + self.app.call_from_thread(self.notify, f"{book['title']}: {error}", title="Skipped", severity="warning") + continue + saved += 1 + self.app.call_from_thread(lambda: self.reload(self.selected_key())) + self.app.call_from_thread(self.notify, f"Saved {saved} of {len(missing)} covers") def action_add_series(self): fields = [FormField("link", "Goodreads series link", placeholder="https://www.goodreads.com/series/92037-wings-of-fire", required=True)] @@ -185,12 +220,21 @@ class BooksPane(ContentPane): @work(thread=True, exclusive=True, group="series") def add_series_books(self, series: dict, choices: dict[str, str]): added = 0 + already_listed = set(books.library_shelves()) + fetched_any = False for entry in series["books"]: shelf = choices.get(entry["goodreadsId"]) if not shelf: continue + needs_fetch = entry["goodreadsId"] not in already_listed + if needs_fetch and fetched_any: + time.sleep(REQUEST_DELAY_SECONDS) + fetched_any = fetched_any or needs_fetch try: book = books.add_series_book(series, entry, shelf) + except goodreads.BlockedError as error: + self.report_blocked(error) + break except (OSError, ValueError) as error: self.app.call_from_thread(self.notify, f"{entry['title']}: {error}", title="Skipped", severity="warning") continue