Cache Goodreads data and slow down bulk requests
This commit is contained in:
1 parent
af73bb5b26
commit
7ef3a29ece
6 files changed
+162
-13
No files matched your search
@@ -22,6 +22,10 @@ REQUEST_HEADERS = {
|
||||
}
|
||||
|
||||
|
||||
class BlockedError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class PageResult:
|
||||
def __init__(self, status: int, final_url: str, text: str):
|
||||
self.status = status
|
||||
@@ -168,10 +172,14 @@ def scrape_book(url: str) -> dict:
|
||||
if not identifier:
|
||||
raise ValueError(f"not a Goodreads book url: {url}")
|
||||
page = download_page(url)
|
||||
raise_if_blocked(page)
|
||||
return parse_book(page.text, identifier)
|
||||
|
||||
|
||||
def raise_if_blocked(page: PageResult):
|
||||
reason = blocked_reason(page.text)
|
||||
if reason:
|
||||
raise ValueError(reason)
|
||||
return parse_book(page.text, identifier)
|
||||
raise BlockedError(f"Goodreads is rate limiting requests ({reason}), wait a few minutes and try again")
|
||||
|
||||
|
||||
def print_diagnostics(url: str, save_path: Path | None):
|
||||
|
||||
Reference in new issue
Block a user