210 lines
8.0 KiB
Python
210 lines
8.0 KiB
Python
import argparse
|
|
import html
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import jsonfile
|
|
from html_page import download_page, meta_tags, page_title, plain_text
|
|
|
|
GAME_URL_PATTERN = re.compile(r"mobygames\.com/game/(\d+)")
|
|
HEADING_PATTERN = re.compile(r"<h1[^>]*>(.*?)</h1>", re.S | re.I)
|
|
METADATA_PATTERN = re.compile(r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>", re.S | re.I)
|
|
LINK_PATTERN = re.compile(r"<a\s[^>]*href=[\"']([^\"']*)[\"'][^>]*>(.*?)</a>", re.S | re.I)
|
|
DESCRIPTION_PATTERN = re.compile(r"<div[^>]*id=[\"']description-text[\"'][^>]*>(.*?)</div>", re.S | re.I)
|
|
COVER_PATTERN = re.compile(r"https://cdn\.mobygames\.com/covers/[^\"'\s)]+")
|
|
SCREENSHOT_PATTERN = re.compile(r"https://cdn\.mobygames\.com/screenshots/[^\"'\s)]+")
|
|
NUMBER_PATTERN = re.compile(r"\d+(?:\.\d+)?")
|
|
PERCENT_PATTERN = re.compile(r"(\d+)\s*%")
|
|
BLOCKED_PAGE_HINTS = ["just a moment", "cf-chl", "captcha", "access denied", "are you a human"]
|
|
DETAILS_FILE = "gameDetails.json"
|
|
DESCRIPTION_MAX_LENGTH = 600
|
|
|
|
|
|
class BlockedError(ValueError):
|
|
pass
|
|
|
|
|
|
def game_id(url: str) -> str | None:
|
|
match = GAME_URL_PATTERN.search(url)
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def blocked_reason(page: str) -> str | None:
|
|
lowered = page.lower()
|
|
hint = next((hint for hint in BLOCKED_PAGE_HINTS if hint in lowered), None)
|
|
if hint and not METADATA_PATTERN.search(page):
|
|
return f"the page looks like a bot check (found '{hint}')"
|
|
return None
|
|
|
|
|
|
def metadata(page: str) -> dict[str, str]:
|
|
fields = {}
|
|
for label, value in METADATA_PATTERN.findall(page):
|
|
name = (plain_text(label) or "").strip().lower()
|
|
if name and name not in fields:
|
|
fields[name] = value
|
|
return fields
|
|
|
|
|
|
def field_starting_with(fields: dict[str, str], *prefixes: str) -> str:
|
|
return next((value for name, value in fields.items() if name.startswith(prefixes)), "")
|
|
|
|
|
|
def link_texts(fragment: str, path_part: str | None = None) -> list[str]:
|
|
names = []
|
|
for href, text in LINK_PATTERN.findall(fragment):
|
|
name = plain_text(text)
|
|
if name and (path_part is None or path_part in href) and name not in names:
|
|
names.append(name)
|
|
return names
|
|
|
|
|
|
def first_number(fragment: str) -> float | None:
|
|
match = NUMBER_PATTERN.search(plain_text(fragment) or "")
|
|
return float(match.group(0)) if match else None
|
|
|
|
|
|
def first_percent(fragment: str) -> int | None:
|
|
match = PERCENT_PATTERN.search(plain_text(fragment) or "")
|
|
return int(match.group(1)) if match else None
|
|
|
|
|
|
def short_description(text: str | None) -> str | None:
|
|
if not text or len(text) <= DESCRIPTION_MAX_LENGTH:
|
|
return text
|
|
return text[:DESCRIPTION_MAX_LENGTH].rsplit(" ", 1)[0] + "..."
|
|
|
|
|
|
def description_of(page: str, meta: dict) -> str | None:
|
|
match = DESCRIPTION_PATTERN.search(page)
|
|
text = plain_text(match.group(1)) if match else None
|
|
return short_description(text or meta.get("og:description") or meta.get("description"))
|
|
|
|
|
|
def title_of(page: str, meta: dict) -> str:
|
|
heading = HEADING_PATTERN.search(page)
|
|
title = plain_text(heading.group(1)) if heading else None
|
|
return title or meta.get("og:title") or page_title(page).split(" - ")[0]
|
|
|
|
|
|
def first_match(pattern: re.Pattern, page: str) -> str | None:
|
|
match = pattern.search(page)
|
|
return html.unescape(match.group(0)) if match else None
|
|
|
|
|
|
def parse_game(page: str, identifier: str, url: str) -> dict:
|
|
meta = meta_tags(page)
|
|
fields = metadata(page)
|
|
released = field_starting_with(fields, "released")
|
|
release_fields = "".join(value for name, value in fields.items() if name.startswith(("released", "releases", "platform")))
|
|
genre_fields = "".join(fields.get(name, "") for name in ["genre", "perspective", "gameplay", "setting"])
|
|
|
|
return {
|
|
"mobygamesId": identifier,
|
|
"url": url,
|
|
"title": title_of(page, meta),
|
|
"released": (link_texts(released) or [plain_text(released)])[0] if released else None,
|
|
"platforms": link_texts(release_fields, "/platform/"),
|
|
"developers": link_texts(field_starting_with(fields, "developer")),
|
|
"publishers": link_texts(field_starting_with(fields, "publisher")),
|
|
"genres": link_texts(genre_fields),
|
|
"mobyScore": first_number(field_starting_with(fields, "moby score")),
|
|
"criticScore": first_percent(field_starting_with(fields, "critic")),
|
|
"playerScore": first_number(field_starting_with(fields, "player")),
|
|
"description": description_of(page, meta),
|
|
"cover": first_match(COVER_PATTERN, page) or meta.get("og:image"),
|
|
"screenshot": first_match(SCREENSHOT_PATTERN, page),
|
|
}
|
|
|
|
|
|
def scrape_game(url: str) -> dict:
|
|
identifier = game_id(url)
|
|
if not identifier:
|
|
raise ValueError(f"not a MobyGames game url: {url}")
|
|
page = download_page(url)
|
|
reason = blocked_reason(page.text)
|
|
if reason:
|
|
raise BlockedError(f"MobyGames blocked the request ({reason}), wait a few minutes and try again")
|
|
return parse_game(page.text, identifier, url)
|
|
|
|
|
|
def save_game(game: dict) -> None:
|
|
details = jsonfile.load_data(DETAILS_FILE, {})
|
|
details[game["mobygamesId"]] = {key: value for key, value in game.items() if key != "mobygamesId"}
|
|
jsonfile.save_data(DETAILS_FILE, dict(sorted(details.items())))
|
|
|
|
|
|
def print_diagnostics(url: str, save_path: Path | None):
|
|
page = download_page(url)
|
|
print(f"Status {page.status}")
|
|
print(f"Final url {page.final_url}")
|
|
print(f"Size {len(page.text):,} characters")
|
|
print(f"Title {page_title(page.text) or '-'}")
|
|
print(f"Fields {', '.join(metadata(page.text)) or '-'}")
|
|
print(f"Meta tags {len(meta_tags(page.text))}")
|
|
print(f"Blocked {blocked_reason(page.text) or 'no sign of a bot check'}")
|
|
if save_path:
|
|
save_path.write_text(page.text, encoding="utf-8")
|
|
print(f"Saved html to {save_path}")
|
|
|
|
|
|
def print_summary(game: dict):
|
|
print(game["title"] or "unknown title")
|
|
print("-" * 60)
|
|
for label, key in [
|
|
("Released", "released"),
|
|
("Platforms", "platforms"),
|
|
("Developers", "developers"),
|
|
("Publishers", "publishers"),
|
|
("Genres", "genres"),
|
|
("Moby score", "mobyScore"),
|
|
("Critics", "criticScore"),
|
|
("Players", "playerScore"),
|
|
("Cover", "cover"),
|
|
("Screenshot", "screenshot"),
|
|
]:
|
|
value = game[key]
|
|
text = ", ".join(value) if isinstance(value, list) else value
|
|
print(f"{label:<11} {text if text not in (None, '') else '-'}")
|
|
if game["description"]:
|
|
print(f"\n{game['description']}")
|
|
missing = [key for key, value in game.items() if value in (None, "", [])]
|
|
if missing:
|
|
print(f"\nNot found on the page: {', '.join(missing)}")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Scrape video game info from MobyGames")
|
|
parser.add_argument("urls", nargs="+", help="MobyGames game urls, for example https://www.mobygames.com/game/223472/hades-ii/")
|
|
parser.add_argument("--json", action="store_true", help="print the raw result as JSON")
|
|
parser.add_argument("--save", action="store_true", help=f"save the result into data/{DETAILS_FILE} for the site")
|
|
parser.add_argument("--debug", action="store_true", help="show what the downloaded page contains instead of parsing it")
|
|
parser.add_argument("--save-html", type=Path, help="with --debug, save the downloaded page to this file")
|
|
arguments = parser.parse_args()
|
|
|
|
exit_code = 0
|
|
for url in arguments.urls:
|
|
try:
|
|
if arguments.debug:
|
|
print_diagnostics(url, arguments.save_html)
|
|
continue
|
|
game = scrape_game(url)
|
|
except (OSError, ValueError) as error:
|
|
print(f"Error for {url}: {error}", file=sys.stderr)
|
|
exit_code = 1
|
|
continue
|
|
if arguments.save:
|
|
save_game(game)
|
|
if arguments.json:
|
|
print(json.dumps(game, indent="\t", ensure_ascii=False))
|
|
else:
|
|
print_summary(game)
|
|
print()
|
|
return exit_code
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|