Cache Goodreads data and slow down bulk requests

This commit is contained in:
SirBlob committed 2026-11-01 21:50:01 -05:00
1 parent af73bb5b26
commit 7ef3a29ece
6 files changed
+162 -13

No files matched your search

+10 -2
View File
@@ -22,6 +22,10 @@ REQUEST_HEADERS = {
}
class BlockedError(ValueError):
pass
class PageResult:
def __init__(self, status: int, final_url: str, text: str):
self.status = status
@@ -168,10 +172,14 @@ def scrape_book(url: str) -> dict:
if not identifier:
raise ValueError(f"not a Goodreads book url: {url}")
page = download_page(url)
raise_if_blocked(page)
return parse_book(page.text, identifier)
def raise_if_blocked(page: PageResult):
reason = blocked_reason(page.text)
if reason:
raise ValueError(reason)
return parse_book(page.text, identifier)
raise BlockedError(f"Goodreads is rate limiting requests ({reason}), wait a few minutes and try again")
def print_diagnostics(url: str, save_path: Path | None):