52 lines
1.8 KiB
Python
52 lines
1.8 KiB
Python
import html
|
|
import re
|
|
import urllib.request
|
|
|
|
META_TAG_PATTERN = re.compile(r"<meta\s[^>]*>", re.S | re.I)
|
|
ATTRIBUTE_PATTERN = re.compile(r"([\w:-]+)=\"([^\"]*)\"|([\w:-]+)='([^']*)'")
|
|
PAGE_TITLE_PATTERN = re.compile(r"<title[^>]*>(.*?)</title>", re.S | re.I)
|
|
TAG_PATTERN = re.compile(r"<[^>]+>")
|
|
|
|
REQUEST_HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
|
"Accept": "text/html,application/xhtml+xml",
|
|
"Accept-Language": "en-US,en;q=0.9",
|
|
}
|
|
|
|
|
|
class PageResult:
|
|
def __init__(self, status: int, final_url: str, text: str):
|
|
self.status = status
|
|
self.final_url = final_url
|
|
self.text = text
|
|
|
|
|
|
def download_page(url: str) -> PageResult:
|
|
request = urllib.request.Request(url, headers=REQUEST_HEADERS)
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
return PageResult(response.status, response.geturl(), response.read().decode("utf-8", errors="replace"))
|
|
|
|
|
|
def page_title(page: str) -> str:
|
|
match = PAGE_TITLE_PATTERN.search(page)
|
|
return html.unescape(match.group(1)).strip() if match else ""
|
|
|
|
|
|
def meta_tags(page: str) -> dict:
|
|
tags = {}
|
|
for tag in META_TAG_PATTERN.findall(page):
|
|
attributes = {}
|
|
for double_name, double_value, single_name, single_value in ATTRIBUTE_PATTERN.findall(tag):
|
|
attributes[(double_name or single_name).lower()] = double_value or single_value
|
|
name = attributes.get("property") or attributes.get("name")
|
|
if name and "content" in attributes:
|
|
tags[name] = html.unescape(attributes["content"])
|
|
return tags
|
|
|
|
|
|
def plain_text(text: str | None) -> str | None:
|
|
if not text:
|
|
return None
|
|
with_breaks = re.sub(r"<br\s*/?>", "\n", text)
|
|
return html.unescape(TAG_PATTERN.sub("", with_breaks)).strip() or None
|