260 lines
10 KiB
Python
260 lines
10 KiB
Python
import argparse
|
|
import datetime
|
|
import html
|
|
import json
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
BOOK_URL_PATTERN = re.compile(r"goodreads\.com/(?:\w+/)?book/show/(\d+)")
|
|
JSON_LD_PATTERN = re.compile(r"<script[^>]*type=[\"']application/ld\+json[\"'][^>]*>(.*?)</script>", re.S | re.I)
|
|
NEXT_DATA_PATTERN = re.compile(r"<script[^>]*id=[\"']__NEXT_DATA__[\"'][^>]*>(.*?)</script>", re.S | re.I)
|
|
META_TAG_PATTERN = re.compile(r"<meta\s[^>]*>", re.S | re.I)
|
|
ATTRIBUTE_PATTERN = re.compile(r"([\w:-]+)=\"([^\"]*)\"|([\w:-]+)='([^']*)'")
|
|
PAGE_TITLE_PATTERN = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
|
|
TAG_PATTERN = re.compile(r"<[^>]+>")
|
|
BLOCKED_PAGE_HINTS = ["captcha", "awswaf", "robot check", "access denied", "are you a human", "unusual traffic"]
|
|
REQUEST_HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
|
"Accept": "text/html,application/xhtml+xml",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
}
|
|
|
|
|
|
class BlockedError(ValueError):
|
|
pass
|
|
|
|
|
|
class PageResult:
|
|
def __init__(self, status: int, final_url: str, text: str):
|
|
self.status = status
|
|
self.final_url = final_url
|
|
self.text = text
|
|
|
|
|
|
def book_id(url: str) -> str | None:
|
|
match = BOOK_URL_PATTERN.search(url)
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def download_page(url: str) -> PageResult:
|
|
request = urllib.request.Request(url, headers=REQUEST_HEADERS)
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
return PageResult(response.status, response.geturl(), response.read().decode("utf-8", errors="replace"))
|
|
|
|
|
|
def page_title(page: str) -> str:
|
|
match = PAGE_TITLE_PATTERN.search(page)
|
|
return html.unescape(match.group(1)).strip() if match else ""
|
|
|
|
|
|
def blocked_reason(page: str) -> str | None:
|
|
lowered = page.lower()
|
|
hint = next((hint for hint in BLOCKED_PAGE_HINTS if hint in lowered), None)
|
|
has_book_data = JSON_LD_PATTERN.search(page) or NEXT_DATA_PATTERN.search(page)
|
|
if hint and not has_book_data:
|
|
return f"the page looks like a bot check (found '{hint}')"
|
|
return None
|
|
|
|
|
|
def json_ld_book(page: str) -> dict:
|
|
for block in JSON_LD_PATTERN.findall(page):
|
|
try:
|
|
data = json.loads(block)
|
|
except ValueError:
|
|
continue
|
|
candidates = data if isinstance(data, list) else data.get("@graph", [data]) if isinstance(data, dict) else []
|
|
for candidate in candidates:
|
|
if isinstance(candidate, dict) and candidate.get("@type") == "Book":
|
|
return candidate
|
|
return {}
|
|
|
|
|
|
def meta_tags(page: str) -> dict:
|
|
tags = {}
|
|
for tag in META_TAG_PATTERN.findall(page):
|
|
attributes = {}
|
|
for double_name, double_value, single_name, single_value in ATTRIBUTE_PATTERN.findall(tag):
|
|
attributes[(double_name or single_name).lower()] = double_value or single_value
|
|
name = attributes.get("property") or attributes.get("name")
|
|
if name and "content" in attributes:
|
|
tags[name] = html.unescape(attributes["content"])
|
|
return tags
|
|
|
|
|
|
def plain_text(text: str | None) -> str | None:
|
|
if not text:
|
|
return None
|
|
with_breaks = re.sub(r"<br\s*/?>", "\n", text)
|
|
return html.unescape(TAG_PATTERN.sub("", with_breaks)).strip() or None
|
|
|
|
|
|
def apollo_state(page: str) -> dict:
|
|
match = NEXT_DATA_PATTERN.search(page)
|
|
if not match:
|
|
return {}
|
|
try:
|
|
return json.loads(match.group(1))["props"]["pageProps"]["apolloState"]
|
|
except (ValueError, KeyError, TypeError):
|
|
return {}
|
|
|
|
|
|
def apollo_book(state: dict, identifier: str) -> tuple[dict, dict]:
|
|
books = [value for value in state.values() if isinstance(value, dict) and value.get("__typename") == "Book"]
|
|
book = next((value for value in books if str(value.get("legacyId")) == identifier), books[0] if books else {})
|
|
work_reference = (book.get("work") or {}).get("__ref")
|
|
work = state.get(work_reference, {}) if work_reference else {}
|
|
return book, work
|
|
|
|
|
|
def contributor_name(state: dict, edge: dict) -> str | None:
|
|
reference = (edge.get("node") or {}).get("__ref")
|
|
return (state.get(reference) or {}).get("name") if reference else None
|
|
|
|
|
|
def book_authors(state: dict, book: dict) -> list[str]:
|
|
edges = [book.get("primaryContributorEdge") or {}]
|
|
edges += [edge for edge in book.get("secondaryContributorEdges") or [] if edge.get("role") == "Author"]
|
|
names = [contributor_name(state, edge) for edge in edges]
|
|
return [html.unescape(name) for name in names if name]
|
|
|
|
|
|
def description_of(book: dict) -> str | None:
|
|
for key, value in book.items():
|
|
if key.startswith("description") and isinstance(value, str):
|
|
return plain_text(value)
|
|
return None
|
|
|
|
|
|
def publication_date(details: dict) -> str | None:
|
|
milliseconds = details.get("publicationTime")
|
|
if not isinstance(milliseconds, (int, float)):
|
|
return None
|
|
return datetime.datetime.fromtimestamp(milliseconds / 1000, tz=datetime.timezone.utc).date().isoformat()
|
|
|
|
|
|
def parse_book(page: str, identifier: str) -> dict:
|
|
linked_data = json_ld_book(page)
|
|
meta = meta_tags(page)
|
|
state = apollo_state(page)
|
|
book, work = apollo_book(state, identifier)
|
|
details = book.get("details") or {}
|
|
work_stats = work.get("stats") or {}
|
|
rating = linked_data.get("aggregateRating") or {}
|
|
linked_authors = linked_data.get("author") or []
|
|
if isinstance(linked_authors, dict):
|
|
linked_authors = [linked_authors]
|
|
authors = book_authors(state, book) or [html.unescape(author["name"]) for author in linked_authors if author.get("name")]
|
|
|
|
return {
|
|
"goodreadsId": identifier,
|
|
"url": f"https://www.goodreads.com/book/show/{identifier}",
|
|
"title": html.unescape(linked_data.get("name") or book.get("title") or meta.get("og:title") or ""),
|
|
"authors": authors,
|
|
"cover": book.get("imageUrl") or linked_data.get("image") or meta.get("og:image"),
|
|
"description": description_of(book) or meta.get("og:description") or meta.get("description"),
|
|
"pages": linked_data.get("numberOfPages") or details.get("numPages"),
|
|
"format": linked_data.get("bookFormat") or details.get("format"),
|
|
"published": publication_date(details),
|
|
"publisher": details.get("publisher"),
|
|
"isbn": linked_data.get("isbn") or details.get("isbn13") or details.get("isbn"),
|
|
"language": linked_data.get("inLanguage") or (details.get("language") or {}).get("name"),
|
|
"genres": [edge["genre"]["name"] for edge in book.get("bookGenres") or [] if (edge.get("genre") or {}).get("name")],
|
|
"averageRating": rating.get("ratingValue") or work_stats.get("averageRating"),
|
|
"ratingCount": rating.get("ratingCount") or work_stats.get("ratingsCount"),
|
|
"reviewCount": rating.get("reviewCount") or work_stats.get("textReviewsCount"),
|
|
}
|
|
|
|
|
|
def scrape_book(url: str) -> dict:
|
|
identifier = book_id(url)
|
|
if not identifier:
|
|
raise ValueError(f"not a Goodreads book url: {url}")
|
|
page = download_page(url)
|
|
raise_if_blocked(page)
|
|
return parse_book(page.text, identifier)
|
|
|
|
|
|
def raise_if_blocked(page: PageResult):
|
|
reason = blocked_reason(page.text)
|
|
if reason:
|
|
raise BlockedError(f"Goodreads is rate limiting requests ({reason}), wait a few minutes and try again")
|
|
|
|
|
|
def print_diagnostics(url: str, save_path: Path | None):
|
|
page = download_page(url)
|
|
print(f"Status {page.status}")
|
|
print(f"Final url {page.final_url}")
|
|
print(f"Size {len(page.text):,} characters")
|
|
print(f"Title {page_title(page.text) or '-'}")
|
|
print(f"JSON-LD {len(JSON_LD_PATTERN.findall(page.text))} blocks")
|
|
print(f"Next data {'found' if NEXT_DATA_PATTERN.search(page.text) else 'missing'}")
|
|
print(f"Meta tags {len(meta_tags(page.text))}")
|
|
print(f"Blocked {blocked_reason(page.text) or 'no sign of a bot check'}")
|
|
if save_path:
|
|
save_path.write_text(page.text, encoding="utf-8")
|
|
print(f"Saved html to {save_path}")
|
|
|
|
|
|
def print_summary(book: dict):
|
|
print(f"{book['title'] or 'unknown title'} by {', '.join(book['authors']) or 'unknown author'}")
|
|
print("-" * 60)
|
|
for label, key in [
|
|
("Cover", "cover"),
|
|
("Pages", "pages"),
|
|
("Format", "format"),
|
|
("Published", "published"),
|
|
("Publisher", "publisher"),
|
|
("ISBN", "isbn"),
|
|
("Language", "language"),
|
|
("Rating", "averageRating"),
|
|
("Ratings", "ratingCount"),
|
|
("Reviews", "reviewCount"),
|
|
]:
|
|
print(f"{label:<10} {book[key] if book[key] is not None else '-'}")
|
|
print(f"{'Genres':<10} {', '.join(book['genres']) or '-'}")
|
|
if book["description"]:
|
|
print(f"\n{book['description'][:400]}{'...' if len(book['description']) > 400 else ''}")
|
|
missing = [key for key, value in book.items() if value in (None, "", [])]
|
|
if missing:
|
|
print(f"\nNot found on the page: {', '.join(missing)}")
|
|
|
|
|
|
def main() -> int:
|
|
import goodreads_series
|
|
|
|
parser = argparse.ArgumentParser(description="Scrape book or series info from Goodreads")
|
|
parser.add_argument("urls", nargs="+", help="Goodreads book or series urls, for example https://www.goodreads.com/book/show/170448.Animal_Farm")
|
|
parser.add_argument("--json", action="store_true", help="print the raw result as JSON")
|
|
parser.add_argument("--debug", action="store_true", help="show what the downloaded page contains instead of parsing it")
|
|
parser.add_argument("--save-html", type=Path, help="with --debug, save the downloaded page to this file")
|
|
arguments = parser.parse_args()
|
|
|
|
exit_code = 0
|
|
for url in arguments.urls:
|
|
is_series = goodreads_series.series_id(url) is not None
|
|
try:
|
|
if arguments.debug:
|
|
print_diagnostics(url, arguments.save_html)
|
|
print(f"React data {len(goodreads_series.REACT_PROPS_PATTERN.findall(download_page(url).text))} blocks")
|
|
continue
|
|
result = goodreads_series.scrape_series(url) if is_series else scrape_book(url)
|
|
except (OSError, ValueError) as error:
|
|
print(f"Error for {url}: {error}", file=sys.stderr)
|
|
exit_code = 1
|
|
continue
|
|
if arguments.json:
|
|
print(json.dumps(result, indent="\t", ensure_ascii=False))
|
|
elif is_series:
|
|
goodreads_series.print_series(result)
|
|
print()
|
|
else:
|
|
print_summary(result)
|
|
print()
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|