Add SQLite library with API, library desktop apps and MobyGames scraper
This commit is contained in:
SirBlob committed
1 parent
6674428346
commit
a16444451a
69 files changed
+2988
-1026
No files matched your search
@@ -0,0 +1,209 @@
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import jsonfile
|
||||
from html_page import download_page, meta_tags, page_title, plain_text
|
||||
|
||||
GAME_URL_PATTERN = re.compile(r"mobygames\.com/game/(\d+)")
|
||||
HEADING_PATTERN = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S | re.I)
|
||||
METADATA_PATTERN = re.compile(r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>", re.S | re.I)
|
||||
LINK_PATTERN = re.compile(r"<a\s[^>]*href=[\"']([^\"']*)[\"'][^>]*>(.*?)</a>", re.S | re.I)
|
||||
DESCRIPTION_PATTERN = re.compile(r"<div[^>]*id=[\"']description-text[\"'][^>]*>(.*?)</div>", re.S | re.I)
|
||||
COVER_PATTERN = re.compile(r"https://cdn\.mobygames\.com/covers/[^\"'\s)]+")
|
||||
SCREENSHOT_PATTERN = re.compile(r"https://cdn\.mobygames\.com/screenshots/[^\"'\s)]+")
|
||||
NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
|
||||
PERCENT_PATTERN = re.compile(r"(\d+)\s*%")
|
||||
BLOCKED_PAGE_HINTS = ["just a moment", "cf-chl", "captcha", "access denied", "are you a human"]
|
||||
DETAILS_FILE = "gameDetails.json"
|
||||
DESCRIPTION_MAX_LENGTH = 600
|
||||
|
||||
|
||||
class BlockedError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def game_id(url: str) -> str | None:
|
||||
match = GAME_URL_PATTERN.search(url)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def blocked_reason(page: str) -> str | None:
|
||||
lowered = page.lower()
|
||||
hint = next((hint for hint in BLOCKED_PAGE_HINTS if hint in lowered), None)
|
||||
if hint and not METADATA_PATTERN.search(page):
|
||||
return f"the page looks like a bot check (found '{hint}')"
|
||||
return None
|
||||
|
||||
|
||||
def metadata(page: str) -> dict[str, str]:
|
||||
fields = {}
|
||||
for label, value in METADATA_PATTERN.findall(page):
|
||||
name = (plain_text(label) or "").strip().lower()
|
||||
if name and name not in fields:
|
||||
fields[name] = value
|
||||
return fields
|
||||
|
||||
|
||||
def field_starting_with(fields: dict[str, str], *prefixes: str) -> str:
|
||||
return next((value for name, value in fields.items() if name.startswith(prefixes)), "")
|
||||
|
||||
|
||||
def link_texts(fragment: str, path_part: str | None = None) -> list[str]:
|
||||
names = []
|
||||
for href, text in LINK_PATTERN.findall(fragment):
|
||||
name = plain_text(text)
|
||||
if name and (path_part is None or path_part in href) and name not in names:
|
||||
names.append(name)
|
||||
return names
|
||||
|
||||
|
||||
def first_number(fragment: str) -> float | None:
|
||||
match = NUMBER_PATTERN.search(plain_text(fragment) or "")
|
||||
return float(match.group(0)) if match else None
|
||||
|
||||
|
||||
def first_percent(fragment: str) -> int | None:
|
||||
match = PERCENT_PATTERN.search(plain_text(fragment) or "")
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def short_description(text: str | None) -> str | None:
|
||||
if not text or len(text) <= DESCRIPTION_MAX_LENGTH:
|
||||
return text
|
||||
return text[:DESCRIPTION_MAX_LENGTH].rsplit(" ", 1)[0] + "..."
|
||||
|
||||
|
||||
def description_of(page: str, meta: dict) -> str | None:
|
||||
match = DESCRIPTION_PATTERN.search(page)
|
||||
text = plain_text(match.group(1)) if match else None
|
||||
return short_description(text or meta.get("og:description") or meta.get("description"))
|
||||
|
||||
|
||||
def title_of(page: str, meta: dict) -> str:
|
||||
heading = HEADING_PATTERN.search(page)
|
||||
title = plain_text(heading.group(1)) if heading else None
|
||||
return title or meta.get("og:title") or page_title(page).split(" - ")[0]
|
||||
|
||||
|
||||
def first_match(pattern: re.Pattern, page: str) -> str | None:
|
||||
match = pattern.search(page)
|
||||
return html.unescape(match.group(0)) if match else None
|
||||
|
||||
|
||||
def parse_game(page: str, identifier: str, url: str) -> dict:
|
||||
meta = meta_tags(page)
|
||||
fields = metadata(page)
|
||||
released = field_starting_with(fields, "released")
|
||||
release_fields = "".join(value for name, value in fields.items() if name.startswith(("released", "releases", "platform")))
|
||||
genre_fields = "".join(fields.get(name, "") for name in ["genre", "perspective", "gameplay", "setting"])
|
||||
|
||||
return {
|
||||
"mobygamesId": identifier,
|
||||
"url": url,
|
||||
"title": title_of(page, meta),
|
||||
"released": (link_texts(released) or [plain_text(released)])[0] if released else None,
|
||||
"platforms": link_texts(release_fields, "/platform/"),
|
||||
"developers": link_texts(field_starting_with(fields, "developer")),
|
||||
"publishers": link_texts(field_starting_with(fields, "publisher")),
|
||||
"genres": link_texts(genre_fields),
|
||||
"mobyScore": first_number(field_starting_with(fields, "moby score")),
|
||||
"criticScore": first_percent(field_starting_with(fields, "critic")),
|
||||
"playerScore": first_number(field_starting_with(fields, "player")),
|
||||
"description": description_of(page, meta),
|
||||
"cover": first_match(COVER_PATTERN, page) or meta.get("og:image"),
|
||||
"screenshot": first_match(SCREENSHOT_PATTERN, page),
|
||||
}
|
||||
|
||||
|
||||
def scrape_game(url: str) -> dict:
|
||||
identifier = game_id(url)
|
||||
if not identifier:
|
||||
raise ValueError(f"not a MobyGames game url: {url}")
|
||||
page = download_page(url)
|
||||
reason = blocked_reason(page.text)
|
||||
if reason:
|
||||
raise BlockedError(f"MobyGames blocked the request ({reason}), wait a few minutes and try again")
|
||||
return parse_game(page.text, identifier, url)
|
||||
|
||||
|
||||
def save_game(game: dict) -> None:
|
||||
details = jsonfile.load_data(DETAILS_FILE, {})
|
||||
details[game["mobygamesId"]] = {key: value for key, value in game.items() if key != "mobygamesId"}
|
||||
jsonfile.save_data(DETAILS_FILE, dict(sorted(details.items())))
|
||||
|
||||
|
||||
def print_diagnostics(url: str, save_path: Path | None):
|
||||
page = download_page(url)
|
||||
print(f"Status {page.status}")
|
||||
print(f"Final url {page.final_url}")
|
||||
print(f"Size {len(page.text):,} characters")
|
||||
print(f"Title {page_title(page.text) or '-'}")
|
||||
print(f"Fields {', '.join(metadata(page.text)) or '-'}")
|
||||
print(f"Meta tags {len(meta_tags(page.text))}")
|
||||
print(f"Blocked {blocked_reason(page.text) or 'no sign of a bot check'}")
|
||||
if save_path:
|
||||
save_path.write_text(page.text, encoding="utf-8")
|
||||
print(f"Saved html to {save_path}")
|
||||
|
||||
|
||||
def print_summary(game: dict):
|
||||
print(game["title"] or "unknown title")
|
||||
print("-" * 60)
|
||||
for label, key in [
|
||||
("Released", "released"),
|
||||
("Platforms", "platforms"),
|
||||
("Developers", "developers"),
|
||||
("Publishers", "publishers"),
|
||||
("Genres", "genres"),
|
||||
("Moby score", "mobyScore"),
|
||||
("Critics", "criticScore"),
|
||||
("Players", "playerScore"),
|
||||
("Cover", "cover"),
|
||||
("Screenshot", "screenshot"),
|
||||
]:
|
||||
value = game[key]
|
||||
text = ", ".join(value) if isinstance(value, list) else value
|
||||
print(f"{label:<11} {text if text not in (None, '') else '-'}")
|
||||
if game["description"]:
|
||||
print(f"\n{game['description']}")
|
||||
missing = [key for key, value in game.items() if value in (None, "", [])]
|
||||
if missing:
|
||||
print(f"\nNot found on the page: {', '.join(missing)}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Scrape video game info from MobyGames")
|
||||
parser.add_argument("urls", nargs="+", help="MobyGames game urls, for example https://www.mobygames.com/game/223472/hades-ii/")
|
||||
parser.add_argument("--json", action="store_true", help="print the raw result as JSON")
|
||||
parser.add_argument("--save", action="store_true", help=f"save the result into data/{DETAILS_FILE} for the site")
|
||||
parser.add_argument("--debug", action="store_true", help="show what the downloaded page contains instead of parsing it")
|
||||
parser.add_argument("--save-html", type=Path, help="with --debug, save the downloaded page to this file")
|
||||
arguments = parser.parse_args()
|
||||
|
||||
exit_code = 0
|
||||
for url in arguments.urls:
|
||||
try:
|
||||
if arguments.debug:
|
||||
print_diagnostics(url, arguments.save_html)
|
||||
continue
|
||||
game = scrape_game(url)
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"Error for {url}: {error}", file=sys.stderr)
|
||||
exit_code = 1
|
||||
continue
|
||||
if arguments.save:
|
||||
save_game(game)
|
||||
if arguments.json:
|
||||
print(json.dumps(game, indent="\t", ensure_ascii=False))
|
||||
else:
|
||||
print_summary(game)
|
||||
print()
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user