diff --git a/content/about.json b/content/about.json index 098987d..d8f0dd0 100644 --- a/content/about.json +++ b/content/about.json @@ -75,6 +75,5 @@ { "name": "Avatar: The Way of Water", "rating": 10, "url": "https://www.imdb.com/title/tt1630029/" }, { "name": "Avatar: Fire and Ash", "rating": 8, "url": "https://www.imdb.com/title/tt1757678/" }, { "name": "Blade Runner 2049", "rating": 10, "url": "https://www.imdb.com/title/tt1856101/" } - ], - "books": [] + ] } diff --git a/content/books.json b/content/books.json new file mode 100644 index 0000000..ff69a78 --- /dev/null +++ b/content/books.json @@ -0,0 +1,3 @@ +[ + { "goodreadsId": "170448", "url": "https://www.goodreads.com/book/show/170448", "title": "Animal Farm", "authors": ["George Orwell"], "cover": "https://m.media-amazon.com/images/S/compressed.photo.goodreads.com/books/1325861570i/170448.jpg", "description": "Animal Farm is the most famous by far of all twentieth-century political allegories. Its account of a group of barnyard animals who revolt against their vicious human master, only to submit to a tyranny erected by their own kind, can fairly be said to have become a universal drama. Orwell is one of the very few modern satirists comparable to Jonathan Swift in power, artistry, and moral authority; in animal farm his spare prose and the logic of his dark comedy brilliantly highlight his stark message.\n \nTaking as his starting point the betrayed promise of the Russian Revolution, Orwell lays out...", "pages": 141, "published": "1996-04-01", "genres": ["Classics", "Fiction", "Dystopia", "Fantasy", "School"], "averageRating": 4.03, "ratingCount": 4853138, "shelf": "read", "rating": 10, "added": "2026-11-01" } +] diff --git a/src/lib/books/books.ts b/src/lib/books/books.ts deleted file mode 100644 index 89bba15..0000000 --- a/src/lib/books/books.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { Book } from '#lib/content'; - -const coverBaseUrl = 'https://covers.openlibrary.org/b/isbn'; -const bookPageBaseUrl = 'https://openlibrary.org/isbn'; - -function cleanIsbn(isbn: string): string { - return isbn.replace(/[^0-9X]/gi, ''); -} - -export function bookCover(book: Book): string | null { - return book.isbn ? `${coverBaseUrl}/${cleanIsbn(book.isbn)}-L.jpg?default=false` : null; -} - -export function bookLink(book: Book): string | null { - if (book.url) return book.url; - return book.isbn ? `${bookPageBaseUrl}/${cleanIsbn(book.isbn)}` : null; -} - -export function sortBooks(books: Book[]): Book[] { - return [...books].sort((first, second) => { - const readingOrder = Number(second.status === 'reading') - Number(first.status === 'reading'); - return readingOrder || (second.rating ?? 0) - (first.rating ?? 0); - }); -} diff --git a/src/lib/components/about/BookCard.svelte b/src/lib/components/about/BookCard.svelte deleted file mode 100644 index ee6b1d3..0000000 --- a/src/lib/components/about/BookCard.svelte +++ /dev/null @@ -1,140 +0,0 @@ - - - -
- {#if cover && !coverFailed} - (coverFailed = true)} /> - {:else} - - - {book.title} - - {/if} - - {#if book.status === 'reading'} - reading - {:else if book.rating} - {book.rating}/10 - {/if} -
- -
- {book.title} - {book.author} -
- - - diff --git a/src/lib/components/about/BookGrid.svelte b/src/lib/components/about/BookGrid.svelte deleted file mode 100644 index 67cd1ba..0000000 --- a/src/lib/components/about/BookGrid.svelte +++ /dev/null @@ -1,36 +0,0 @@ - - -
- {#each visibleBooks as book (`${book.title}-${book.author}`)} - - {/each} -
- - - - diff --git a/src/lib/components/about/books/BookCard.svelte b/src/lib/components/about/books/BookCard.svelte new file mode 100644 index 0000000..ca708c6 --- /dev/null +++ b/src/lib/components/about/books/BookCard.svelte @@ -0,0 +1,159 @@ + + + +
+ {#if book.cover && !coverFailed} + (coverFailed = true)} /> + {:else} + + + {book.title} + + {/if} + + {#if hasOwnRating} + {book.rating}/10 + {:else if book.averageRating} + {book.averageRating} + {/if} + + {#if book.description} +

{book.description}

+ {/if} +
+ + {book.title} + {book.authors.join(', ')} + {#if book.genres.length} + {book.genres.slice(0, 3).join(' · ')} + {/if} + + + diff --git a/src/lib/components/about/books/BookShelfGrid.svelte b/src/lib/components/about/books/BookShelfGrid.svelte new file mode 100644 index 0000000..6be73da --- /dev/null +++ b/src/lib/components/about/books/BookShelfGrid.svelte @@ -0,0 +1,67 @@ + + +

{title} {books.length}

+
+ {#each shownBooks as book (book.goodreadsId)} + + {/each} +
+ +{#if books.length > collapsedCount} + +{/if} + + diff --git a/src/lib/components/about/books/BooksSection.svelte b/src/lib/components/about/books/BooksSection.svelte new file mode 100644 index 0000000..eeaa477 --- /dev/null +++ b/src/lib/components/about/books/BooksSection.svelte @@ -0,0 +1,180 @@ + + +
+ +
+ {#each shelfCounts as shelf (shelf.label)} +
{shelf.label}: {shelf.count}
+ {/each} + {#each rows as [label, value] (label)} +
{label}: {value}
+ {/each} + + {#if shownCounts.length} + + {/if} +
+ + + {#if reading.length} + +
+ {#each reading as book (book.goodreadsId)} + + {#if book.cover} + {book.title} + {:else} + + {/if} + + {book.title} + {book.authors.join(', ')} + + {[book.pages ? `${book.pages} pages` : '', book.published?.slice(0, 4) ?? ''].filter(Boolean).join(' · ')} + + + + {/each} +
+ + {/if} +
+ +{#if read.length} + +{/if} + +{#if wantToRead.length} + +{/if} + + diff --git a/src/lib/content/index.ts b/src/lib/content/index.ts index fc4d81a..cf452ac 100644 --- a/src/lib/content/index.ts +++ b/src/lib/content/index.ts @@ -11,9 +11,11 @@ import hackathonsData from '#content/projects/hackathons.json'; import researchData from '#content/projects/research.json'; import organizationsData from '#content/projects/organizations.json'; import tcgDecksData from '#content/tcgDecks.json'; +import booksData from '#content/books.json'; import type { AboutContent, BlogSettings, + Book, DesktopContent, Experience, Leadership, @@ -37,6 +39,7 @@ export const blogSettings: BlogSettings = blogData; export const playlist: Playlist = musicData; export const researchGroups: ResearchGroup[] = researchData; export const tcgDecks: TcgDeck[] = tcgDecksData as TcgDeck[]; +export const books: Book[] = booksData as Book[]; function featuredFirst(projects: Project[]): Project[] { return [...projects].sort((first, second) => Number(!!second.featured) - Number(!!first.featured)); diff --git a/src/lib/content/types.ts b/src/lib/content/types.ts index 14c170c..952a074 100644 --- a/src/lib/content/types.ts +++ b/src/lib/content/types.ts @@ -139,15 +139,23 @@ export interface RatedTitle { url: string; } -export type BookStatus = 'read' | 'reading'; +export type BookShelf = 'read' | 'reading' | 'want-to-read'; export interface Book { + goodreadsId: string; + url: string; title: string; - author: string; + authors: string[]; + cover: string | null; + description: string | null; + pages: number | null; + published: string | null; + genres: string[]; + averageRating: number | null; + ratingCount: number | null; + shelf: BookShelf; rating?: number; - isbn?: string; - url?: string; - status?: BookStatus; + added?: string; } export interface LabeledValue { @@ -172,7 +180,6 @@ export interface AboutContent { games: Game[]; shows: RatedTitle[]; movies: RatedTitle[]; - books?: Book[]; } export type GameName = 'snake' | 'breakout'; diff --git a/src/routes/about/+page.svelte b/src/routes/about/+page.svelte index a87d8ba..5b94466 100644 --- a/src/routes/about/+page.svelte +++ b/src/routes/about/+page.svelte @@ -1,5 +1,5 @@ ", re.S | re.I) +NEXT_DATA_PATTERN = re.compile(r"]*id=[\"']__NEXT_DATA__[\"'][^>]*>(.*?)", re.S | re.I) +META_TAG_PATTERN = re.compile(r"]*>", re.S | re.I) +ATTRIBUTE_PATTERN = re.compile(r"([\w:-]+)=\"([^\"]*)\"|([\w:-]+)='([^']*)'") +PAGE_TITLE_PATTERN = re.compile(r"]*>(.*?)", re.S | re.I) +TAG_PATTERN = re.compile(r"<[^>]+>") +BLOCKED_PAGE_HINTS = ["captcha", "awswaf", "robot check", "access denied", "are you a human", "unusual traffic"] +REQUEST_HEADERS = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "text/html,application/xhtml+xml", + "Accept-Language": "en-US,en;q=0.9", +} + + +class PageResult: + def __init__(self, status: int, final_url: str, text: str): + self.status = status + self.final_url = final_url + self.text = text + + +def book_id(url: str) -> str | None: + match = BOOK_URL_PATTERN.search(url) + return match.group(1) if match else None + + +def download_page(url: str) -> PageResult: + request = urllib.request.Request(url, headers=REQUEST_HEADERS) + with urllib.request.urlopen(request, timeout=15) as response: + return PageResult(response.status, response.geturl(), response.read().decode("utf-8", errors="replace")) + + +def page_title(page: str) -> str: + match = PAGE_TITLE_PATTERN.search(page) + return html.unescape(match.group(1)).strip() if match else "" + + +def blocked_reason(page: str) -> str | None: + lowered = page.lower() + hint = next((hint for hint in BLOCKED_PAGE_HINTS if hint in lowered), None) + has_book_data = JSON_LD_PATTERN.search(page) or NEXT_DATA_PATTERN.search(page) + if hint and not has_book_data: + return f"the page looks like a bot check (found '{hint}')" + return None + + +def json_ld_book(page: str) -> dict: + for block in JSON_LD_PATTERN.findall(page): + try: + data = json.loads(block) + except ValueError: + continue + candidates = data if isinstance(data, list) else data.get("@graph", [data]) if isinstance(data, dict) else [] + for candidate in candidates: + if isinstance(candidate, dict) and candidate.get("@type") == "Book": + return candidate + return {} + + +def meta_tags(page: str) -> dict: + tags = {} + for tag in META_TAG_PATTERN.findall(page): + attributes = {} + for double_name, double_value, single_name, single_value in ATTRIBUTE_PATTERN.findall(tag): + attributes[(double_name or single_name).lower()] = double_value or single_value + name = attributes.get("property") or attributes.get("name") + if name and "content" in attributes: + tags[name] = html.unescape(attributes["content"]) + return tags + + +def plain_text(text: str | None) -> str | None: + if not text: + return None + with_breaks = re.sub(r"", "\n", text) + return html.unescape(TAG_PATTERN.sub("", with_breaks)).strip() or None + + +def apollo_state(page: str) -> dict: + match = NEXT_DATA_PATTERN.search(page) + if not match: + return {} + try: + return json.loads(match.group(1))["props"]["pageProps"]["apolloState"] + except (ValueError, KeyError, TypeError): + return {} + + +def apollo_book(state: dict, identifier: str) -> tuple[dict, dict]: + books = [value for value in state.values() if isinstance(value, dict) and value.get("__typename") == "Book"] + book = next((value for value in books if str(value.get("legacyId")) == identifier), books[0] if books else {}) + work_reference = (book.get("work") or {}).get("__ref") + work = state.get(work_reference, {}) if work_reference else {} + return book, work + + +def contributor_name(state: dict, edge: dict) -> str | None: + reference = (edge.get("node") or {}).get("__ref") + return (state.get(reference) or {}).get("name") if reference else None + + +def book_authors(state: dict, book: dict) -> list[str]: + edges = [book.get("primaryContributorEdge") or {}] + edges += [edge for edge in book.get("secondaryContributorEdges") or [] if edge.get("role") == "Author"] + names = [contributor_name(state, edge) for edge in edges] + return [html.unescape(name) for name in names if name] + + +def description_of(book: dict) -> str | None: + for key, value in book.items(): + if key.startswith("description") and isinstance(value, str): + return plain_text(value) + return None + + +def publication_date(details: dict) -> str | None: + milliseconds = details.get("publicationTime") + if not isinstance(milliseconds, (int, float)): + return None + return datetime.datetime.fromtimestamp(milliseconds / 1000, tz=datetime.timezone.utc).date().isoformat() + + +def parse_book(page: str, identifier: str) -> dict: + linked_data = json_ld_book(page) + meta = meta_tags(page) + state = apollo_state(page) + book, work = apollo_book(state, identifier) + details = book.get("details") or {} + work_stats = work.get("stats") or {} + rating = linked_data.get("aggregateRating") or {} + linked_authors = linked_data.get("author") or [] + if isinstance(linked_authors, dict): + linked_authors = [linked_authors] + authors = book_authors(state, book) or [html.unescape(author["name"]) for author in linked_authors if author.get("name")] + + return { + "goodreadsId": identifier, + "url": f"https://www.goodreads.com/book/show/{identifier}", + "title": html.unescape(linked_data.get("name") or book.get("title") or meta.get("og:title") or ""), + "authors": authors, + "cover": book.get("imageUrl") or linked_data.get("image") or meta.get("og:image"), + "description": description_of(book) or meta.get("og:description") or meta.get("description"), + "pages": linked_data.get("numberOfPages") or details.get("numPages"), + "format": linked_data.get("bookFormat") or details.get("format"), + "published": publication_date(details), + "publisher": details.get("publisher"), + "isbn": linked_data.get("isbn") or details.get("isbn13") or details.get("isbn"), + "language": linked_data.get("inLanguage") or (details.get("language") or {}).get("name"), + "genres": [edge["genre"]["name"] for edge in book.get("bookGenres") or [] if (edge.get("genre") or {}).get("name")], + "averageRating": rating.get("ratingValue") or work_stats.get("averageRating"), + "ratingCount": rating.get("ratingCount") or work_stats.get("ratingsCount"), + "reviewCount": rating.get("reviewCount") or work_stats.get("textReviewsCount"), + } + + +def scrape_book(url: str) -> dict: + identifier = book_id(url) + if not identifier: + raise ValueError(f"not a Goodreads book url: {url}") + page = download_page(url) + reason = blocked_reason(page.text) + if reason: + raise ValueError(reason) + return parse_book(page.text, identifier) + + +def print_diagnostics(url: str, save_path: Path | None): + page = download_page(url) + print(f"Status {page.status}") + print(f"Final url {page.final_url}") + print(f"Size {len(page.text):,} characters") + print(f"Title {page_title(page.text) or '-'}") + print(f"JSON-LD {len(JSON_LD_PATTERN.findall(page.text))} blocks") + print(f"Next data {'found' if NEXT_DATA_PATTERN.search(page.text) else 'missing'}") + print(f"Meta tags {len(meta_tags(page.text))}") + print(f"Blocked {blocked_reason(page.text) or 'no sign of a bot check'}") + if save_path: + save_path.write_text(page.text, encoding="utf-8") + print(f"Saved html to {save_path}") + + +def print_summary(book: dict): + print(f"{book['title'] or 'unknown title'} by {', '.join(book['authors']) or 'unknown author'}") + print("-" * 60) + for label, key in [ + ("Cover", "cover"), + ("Pages", "pages"), + ("Format", "format"), + ("Published", "published"), + ("Publisher", "publisher"), + ("ISBN", "isbn"), + ("Language", "language"), + ("Rating", "averageRating"), + ("Ratings", "ratingCount"), + ("Reviews", "reviewCount"), + ]: + print(f"{label:<10} {book[key] if book[key] is not None else '-'}") + print(f"{'Genres':<10} {', '.join(book['genres']) or '-'}") + if book["description"]: + print(f"\n{book['description'][:400]}{'...' if len(book['description']) > 400 else ''}") + missing = [key for key, value in book.items() if value in (None, "", [])] + if missing: + print(f"\nNot found on the page: {', '.join(missing)}") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Scrape book info from a Goodreads book page") + parser.add_argument("urls", nargs="+", help="Goodreads book urls, for example https://www.goodreads.com/book/show/170448.Animal_Farm") + parser.add_argument("--json", action="store_true", help="print the raw result as JSON") + parser.add_argument("--debug", action="store_true", help="show what the downloaded page contains instead of parsing it") + parser.add_argument("--save-html", type=Path, help="with --debug, save the downloaded page to this file") + arguments = parser.parse_args() + + exit_code = 0 + for url in arguments.urls: + try: + if arguments.debug: + print_diagnostics(url, arguments.save_html) + continue + book = scrape_book(url) + except (OSError, ValueError) as error: + print(f"Error for {url}: {error}", file=sys.stderr) + exit_code = 1 + continue + if arguments.json: + print(json.dumps(book, indent="\t", ensure_ascii=False)) + else: + print_summary(book) + print() + return exit_code + + +if __name__ == "__main__": + sys.exit(main())