Add library push to website from the content editor

.env.example

@@ -7,3 +7,5 @@ IGDB_CLIENT_SECRET=
 LASTFM_API_KEY=
 LIBRARY_API_KEY=
 LIBRARY_API_URL=http://localhost:5000
+LIBRARY_PUSH_URL=https://dev.sirblob.co
+LIBRARY_PUSH_KEY=

README.md

@@ -105,6 +105,8 @@ Copy `.env.example` to `.env` and fill in what you need. Every value is optional
 | `LASTFM_API_KEY` | Collecting Last.fm scrobbles for the listening stats |
 | `LIBRARY_API_KEY` | Secret the content editor sends to change shows, movies, books and games. Set the same value on the server and locally |
 | `LIBRARY_API_URL` | Site the content editor talks to, `http://localhost:5000` by default |
+| `LIBRARY_PUSH_URL` | Live site the Push to website button copies the library to, for example `https://dev.sirblob.co` |
+| `LIBRARY_PUSH_KEY` | `LIBRARY_API_KEY` of that live site, when it differs from the local one |
 | `LIBRARY_DATA_DIR` | Folder for `library.db`, `data/library` by default |
 | `LASTFM_DATA_DIR` | Folder for `lastfm.db`, `data/lastfm` by default |
 
@@ -123,6 +125,7 @@ Personal content and config live in `content/`. Shows, movies, books and games a
 | `DELETE /api/library/books/<goodreadsId>` | Remove a book |
 | `PUT /api/library/games/<igdbId>` | Add or change a game |
 | `DELETE /api/library/games/<igdbId>` | Remove a game |
+| `GET /api/library/export` | Every entry with all its fields and image sources, needs the API key |
 | `GET /api/library/images/<id>` | A saved poster, cover or game image |
 
 Changes need `Authorization: Bearer <LIBRARY_API_KEY>`. A `PUT` only changes the fields it sends, and `null` clears one. When a status changes to watching, reading or playing the start date is set, and to watched, read or played the finish date, unless the request sends its own. Posters, covers and game images are downloaded by the server and kept in the database. A site path such as `/img/minecraft.png` is used as it is.
@@ -157,9 +160,10 @@ bun run content
 | `u` / `U` | Refresh the selected book, or every book, from Goodreads; `u` also refreshes a show or movie from OMDb, or a game from IGDB |
 | `C` | Save every missing book cover, or every missing show and movie poster and details, into the library |
 | `r` | Reload from disk or the library |
+| `P` | Push the whole library to the live site |
 | `q` | Quit |
 
-The Shows, Movies, Books and Games tabs read and write the library through the API at `LIBRARY_API_URL` using `LIBRARY_API_KEY`, so point it at the live site to update it from anywhere. Show and movie titles are looked up on OMDb with `OMDB_API_KEY` from `.env` when added; fill in the name field to skip the lookup. Books are scraped from Goodreads when added. To check a Goodreads link on its own, run `python3 tools/content/goodreads.py <url>`.
+The Shows, Movies, Books and Games tabs read and write the library through the API at `LIBRARY_API_URL` using `LIBRARY_API_KEY`, so point it at the live site to update it from anywhere. To work on a local copy instead, keep `LIBRARY_API_URL` on `localhost` and press `P` (or run `bun run library:push`) to copy every show, movie, book and game to `LIBRARY_PUSH_URL`. It asks first, overwrites the entries there with the local ones, removes entries that only exist there, and refuses to push an empty library. Images are downloaded again by the live site from their original sources. Show and movie titles are looked up on OMDb with `OMDB_API_KEY` from `.env` when added; fill in the name field to skip the lookup. Books are scraped from Goodreads when added. To check a Goodreads link on its own, run `python3 tools/content/goodreads.py <url>`.
 
 Games are looked up on IGDB with `IGDB_CLIENT_ID` and `IGDB_CLIENT_SECRET`. A Steam or Xbox link is matched to its IGDB entry directly; any other store link (Nintendo, Epic, GOG and so on) is searched by the name in the link, or by the name you enter. An IGDB link such as `https://www.igdb.com/games/hades-ii` always matches exactly. IGDB fills in the release date, platforms, developers, publishers, genres, summary, rating and a 16:9 artwork image; enter your own image to use it instead. To check a lookup on its own, run `python3 tools/content/igdb.py <link or name>`.
 

package.json

@@ -12,6 +12,7 @@
 		"wallpapers": "tools/content/.venv/bin/python tools/content/wallpapers.py",
 		"games:import": "tools/content/.venv/bin/python tools/content/games_import.py",
 		"files": "tools/content/.venv/bin/python tools/content/static_files.py",
+		"library:push": "tools/content/.venv/bin/python tools/content/library_push.py",
 		"content:setup":"python3 -m venv tools/content/.venv && tools/content/.venv/bin/pip install -r tools/content/requirements.txt",
 		"prepare": "svelte-kit sync || echo ''",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",

src/lib/server/library/books.ts

@@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit';
 import type { Book, BookSeries, BookShelf } from '#lib/content';
 import { timestamp, type SqlDatabase } from './database';
 import { hasImage, imageUrl, removeImage, syncImage } from './images';
-import { parseList, readChoice, readHistory, readNumber, readRating, readText, readTextList, type RequestBody } from './input';
+import { parseList, readChoice, readHistory, readNumber, readRating, readText, readTextList, type ExportedEntry, type RequestBody } from './input';
 
 interface BookRow {
 	goodreads_id: string;
@@ -74,6 +74,30 @@ export function listBooks(database: SqlDatabase, imageIds: Set<string>): Book[]
 	return rows.map((row) => toBook(row, imageIds.has(coverImageId(row.goodreads_id))));
 }
 
+export function exportBooks(database: SqlDatabase): ExportedEntry[] {
+	const rows = database.prepare('SELECT * FROM books ORDER BY goodreads_id').all() as BookRow[];
+	return rows.map((row) => ({
+		id: row.goodreads_id,
+		fields: {
+			title: row.title,
+			authors: parseList(row.authors),
+			description: row.description,
+			pages: row.pages,
+			published: row.published,
+			genres: parseList(row.genres),
+			averageRating: row.average_rating,
+			ratingCount: row.rating_count,
+			coverSource: row.cover_source,
+			shelf: row.shelf,
+			rating: row.rating,
+			series: row.series_name ? { name: row.series_name, url: row.series_url, position: row.series_position } : null,
+			added: row.added,
+			started: row.started,
+			finished: row.finished
+		}
+	}));
+}
+
 function readSeries(body: RequestBody, existing: BookRow | null): Pick<BookRow, 'series_name' | 'series_url' | 'series_position'> {
 	const current = {
 		series_name: existing?.series_name ?? null,

src/lib/server/library/games.ts

@@ -3,7 +3,7 @@ import type { Game, PlayStatus } from '#lib/content';
 import { likedLimit } from '#lib/library/gameProfile';
 import { timestamp, type SqlDatabase } from './database';
 import { hasImage, imageUrl, removeImage, syncImage } from './images';
-import { parseList, readChoice, readFlag, readHistory, readNumber, readRating, readText, readTextList, type RequestBody } from './input';
+import { parseList, readChoice, readFlag, readHistory, readNumber, readRating, readText, readTextList, type ExportedEntry, type RequestBody } from './input';
 
 interface GameRow {
 	igdb_id: string;
@@ -77,6 +77,36 @@ export function listGames(database: SqlDatabase, imageIds: Set<string>): Game[]
 	return rows.map((row) => toGame(row, imageIds.has(gameImageId(row.igdb_id))));
 }
 
+export function exportGames(database: SqlDatabase): ExportedEntry[] {
+	const rows = database.prepare('SELECT * FROM games ORDER BY igdb_id').all() as GameRow[];
+	return rows.map((row) => ({
+		id: row.igdb_id,
+		fields: {
+			name: row.name,
+			url: row.store_url,
+			igdbUrl: row.igdb_url,
+			status: row.status,
+			rating: row.rating,
+			released: row.released,
+			platforms: parseList(row.platforms),
+			developers: parseList(row.developers),
+			publishers: parseList(row.publishers),
+			genres: parseList(row.genres),
+			summary: row.summary,
+			score: row.score,
+			imageSource: row.image_source,
+			coverSource: row.cover_source,
+			favorite: !!row.favorite,
+			liked: !!row.liked,
+			tags: parseList(row.tags),
+			note: row.note,
+			added: row.added,
+			started: row.started,
+			finished: row.finished
+		}
+	}));
+}
+
 function readList(body: RequestBody, key: string, current: string | undefined): string {
 	return JSON.stringify(readTextList(body, key, parseList(current ?? '[]')));
 }

src/lib/server/library/index.ts

@@ -1,12 +1,24 @@
 import type { LibraryData } from '#lib/library/types';
 import { libraryDatabase } from './database';
 import { storedImageIds } from './images';
-import { listMedia } from './media';
+import { exportMedia, listMedia } from './media';
-import { listBooks } from './books';
+import { exportBooks, listBooks } from './books';
-import { listGames } from './games';
+import { exportGames, listGames } from './games';
+import type { ExportedEntry } from './input';
 
 export async function getLibrary(): Promise<LibraryData> {
 	const database = await libraryDatabase();
 	const imageIds = storedImageIds(database);
 	return { ...listMedia(database, imageIds), books: listBooks(database, imageIds), games: listGames(database, imageIds) };
 }
+
+export interface LibraryExport {
+	media: ExportedEntry[];
+	books: ExportedEntry[];
+	games: ExportedEntry[];
+}
+
+export async function exportLibrary(): Promise<LibraryExport> {
+	const database = await libraryDatabase();
+	return { media: exportMedia(database), books: exportBooks(database), games: exportGames(database) };
+}

src/lib/server/library/input.ts

@@ -3,6 +3,11 @@ import { today } from './database';
 
 export type RequestBody = Record<string, unknown>;
 
+export interface ExportedEntry {
+	id: string;
+	fields: RequestBody;
+}
+
 const datePattern = /^\d{4}-\d{2}-\d{2}$/;
 
 export async function readBody(request: Request): Promise<RequestBody> {

src/lib/server/library/media.ts

@@ -4,7 +4,7 @@ import type { MediaDetails, MediaKind } from '#lib/media/types';
 import type { LibraryMedia } from '#lib/library/types';
 import { timestamp, type SqlDatabase } from './database';
 import { hasImage, imageUrl, removeImage, syncImage } from './images';
-import { readChoice, readHistory, readRating, readText, type RequestBody } from './input';
+import { readChoice, readHistory, readRating, readText, type ExportedEntry, type RequestBody } from './input';
 
 interface MediaRow {
 	imdb_id: string;
@@ -83,6 +83,28 @@ export function listMedia(database: SqlDatabase, imageIds: Set<string>): Library
 	};
 }
 
+export function exportMedia(database: SqlDatabase): ExportedEntry[] {
+	const rows = database.prepare('SELECT * FROM media ORDER BY imdb_id').all() as MediaRow[];
+	return rows.map((row) => ({
+		id: row.imdb_id,
+		fields: {
+			kind: row.kind,
+			name: row.name,
+			status: row.status,
+			rating: row.rating,
+			collection: row.collection,
+			year: row.year,
+			genre: row.genre,
+			runtime: row.runtime,
+			plot: row.plot,
+			posterSource: row.poster_source,
+			added: row.added,
+			started: row.started,
+			finished: row.finished
+		}
+	}));
+}
+
 function mergeRow(imdbId: string, existing: MediaRow | null, body: RequestBody): MediaRow {
 	const name = readText(body, 'name', existing?.name ?? null);
 	if (!name) error(400, 'name is required');

src/routes/api/library/export/+server.ts

@@ -0,0 +1,9 @@
+import { json } from '@sveltejs/kit';
+import { exportLibrary } from '#lib/server/library';
+import { requireApiKey } from '#lib/server/library/auth';
+import type { RequestHandler } from './$types';
+
+export const GET: RequestHandler = async ({ request }) => {
+	requireApiKey(request);
+	return json(await exportLibrary(), { headers: { 'Cache-Control': 'no-store' } });
+};

tools/content/app.py

@@ -1,12 +1,14 @@
 import sys
 
-from textual import on
+from textual import on, work
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.widgets import DataTable, Footer, Header, TabbedContent, TabPane
 
+import library_push
 import portfolio
 from tui.books_pane import BooksPane
+from tui.dialogs import ConfirmDialog
 from tui.games_pane import GamesPane
 from tui.media_pane import MediaPane
 from tui.music_pane import MusicPane
@@ -41,6 +43,7 @@ class ContentApp(App):
     BINDINGS = [
         Binding("q", "quit", "Quit"),
         Binding("t", "next_theme", "Theme"),
+        Binding("P", "push_library", "Push to website"),
         *[Binding(tab_key(number), f"show_tab('{tab_id}')", label, show=False) for number, (tab_id, label) in enumerate(TABS, start=1)],
     ]
 
@@ -97,6 +100,36 @@ class ContentApp(App):
     def on_mount(self):
         self.focus_active_table()
 
+    def action_push_library(self):
+        self.notify("Comparing the library with the website...")
+        self.prepare_push()
+
+    @work(thread=True, exclusive=True, group="push")
+    def prepare_push(self):
+        try:
+            plan = library_push.plan_push()
+        except (OSError, ValueError) as error:
+            self.call_from_thread(self.notify, str(error), title="Push", severity="error", timeout=10)
+            return
+        self.call_from_thread(self.confirm_push, plan)
+
+    def confirm_push(self, plan: library_push.PushPlan):
+        def confirmed(should_push: bool | None):
+            if should_push:
+                self.push_library(plan)
+
+        self.push_screen(ConfirmDialog(plan.describe(), "Push", "primary"), confirmed)
+
+    @work(thread=True, exclusive=True, group="push")
+    def push_library(self, plan: library_push.PushPlan):
+        progress = lambda message: self.call_from_thread(self.notify, message)
+        result = library_push.run_push(plan, progress)
+        for message in result.errors[:5]:
+            self.call_from_thread(self.notify, message, title="Push", severity="warning", timeout=10)
+        summary = f"Pushed {result.pushed} and removed {result.removed} on {plan.target.url}"
+        severity = "warning" if result.errors else "information"
+        self.call_from_thread(self.notify, f"{summary}, {len(result.errors)} failed", title="Push", severity=severity, timeout=10)
+
     def action_show_tab(self, tab_id: str):
         self.query_one(TabbedContent).active = tab_id
 

tools/content/library_api.py

@@ -14,15 +14,30 @@ class LibraryError(ValueError):
     pass
 
 
+class Site:
+    def __init__(self, url: str, key: str | None):
+        self.url = url.rstrip("/")
+        self.key = key
+
+    def require_key(self) -> str:
+        if not self.key:
+            raise LibraryError(f"Set an API key in .env to change the library at {self.url}")
+        return self.key
+
+
 def api_url() -> str:
     return (env_value("LIBRARY_API_URL") or DEFAULT_API_URL).rstrip("/")
 
 
-def api_key() -> str:
+def local_site() -> Site:
-    key = env_value("LIBRARY_API_KEY")
+    return Site(api_url(), env_value("LIBRARY_API_KEY"))
-    if not key:
+
-        raise LibraryError("Set LIBRARY_API_KEY in .env to change the library")
+
-    return key
+def push_site() -> Site:
+    url = env_value("LIBRARY_PUSH_URL")
+    if not url:
+        raise LibraryError("Set LIBRARY_PUSH_URL in .env to the website to push to, for example https://dev.sirblob.co")
+    return Site(url, env_value("LIBRARY_PUSH_KEY") or env_value("LIBRARY_API_KEY"))
 
 
 def error_message(error: urllib.error.HTTPError) -> str:
@@ -32,22 +47,23 @@ def error_message(error: urllib.error.HTTPError) -> str:
         return str(error.reason)
 
 
-def send(method: str, path: str, body: dict | None = None, signed: bool = True):
+def send(method: str, path: str, body: dict | None = None, signed: bool = True, site: Site | None = None):
+    site = site or local_site()
     headers = {"Accept": "application/json"}
     data = None
     if signed:
-        headers["Authorization"] = f"Bearer {api_key()}"
+        headers["Authorization"] = f"Bearer {site.require_key()}"
     if body is not None:
         headers["Content-Type"] = "application/json"
         data = json.dumps(body).encode("utf-8")
-    request = urllib.request.Request(f"{api_url()}{path}", data=data, headers=headers, method=method)
+    request = urllib.request.Request(f"{site.url}{path}", data=data, headers=headers, method=method)
     try:
         with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT_SECONDS) as response:
             text = response.read().decode("utf-8")
     except urllib.error.HTTPError as error:
-        raise LibraryError(f"{api_url()} said {error.code}: {error_message(error)}") from error
+        raise LibraryError(f"{site.url} said {error.code}: {error_message(error)}") from error
     except urllib.error.URLError as error:
-        raise LibraryError(f"Could not reach {api_url()}: {error.reason}") from error
+        raise LibraryError(f"Could not reach {site.url}: {error.reason}") from error
     return json.loads(text) if text else None
 
 
@@ -59,6 +75,10 @@ def fetch_library() -> dict:
     return send("GET", "/api/library", signed=False)
 
 
+def fetch_export(site: Site) -> dict:
+    return send("GET", "/api/library/export", site=site)
+
+
 def save_media(imdb_id: str, fields: dict) -> dict:
     return send("PUT", f"/api/library/media/{quoted(imdb_id)}", fields)
 

tools/content/library_push.py

@@ -0,0 +1,97 @@
+import sys
+import urllib.parse
+from dataclasses import dataclass, field
+from typing import Callable
+
+import library_api
+
+SECTIONS = {
+    "media": {"label": "shows and movies", "path": "/api/library/media/"},
+    "books": {"label": "books", "path": "/api/library/books/"},
+    "games": {"label": "games", "path": "/api/library/games/"},
+}
+
+
+@dataclass
+class PushPlan:
+    source: library_api.Site
+    target: library_api.Site
+    entries: dict[str, list[dict]]
+    removals: dict[str, list[str]]
+
+    def describe(self) -> str:
+        pushed = ", ".join(f"{len(self.entries[name])} {section['label']}" for name, section in SECTIONS.items())
+        removed = sum(len(identifiers) for identifiers in self.removals.values())
+        removal_text = f" and remove {removed} entries that only exist there" if removed else ""
+        return f"Push {pushed} from {self.source.url} to {self.target.url}{removal_text}?"
+
+
+@dataclass
+class PushResult:
+    pushed: int = 0
+    removed: int = 0
+    errors: list[str] = field(default_factory=list)
+
+
+def entry_path(section: str, identifier: str) -> str:
+    return f"{SECTIONS[section]['path']}{urllib.parse.quote(identifier, safe='')}"
+
+
+def plan_push() -> PushPlan:
+    source = library_api.local_site()
+    target = library_api.push_site()
+    if source.url == target.url:
+        raise library_api.LibraryError("LIBRARY_PUSH_URL is the same site as LIBRARY_API_URL")
+
+    local = library_api.fetch_export(source)
+    if not any(local[section] for section in SECTIONS):
+        raise library_api.LibraryError(f"The library at {source.url} is empty, so nothing was pushed")
+    remote = library_api.fetch_export(target)
+
+    removals = {}
+    for section in SECTIONS:
+        local_ids = {entry["id"] for entry in local[section]}
+        removals[section] = [entry["id"] for entry in remote[section] if entry["id"] not in local_ids]
+    return PushPlan(source, target, {section: local[section] for section in SECTIONS}, removals)
+
+
+def run_push(plan: PushPlan, on_progress: Callable[[str], None] = lambda message: None) -> PushResult:
+    result = PushResult()
+    for section, identifiers in plan.removals.items():
+        for identifier in identifiers:
+            try:
+                library_api.send("DELETE", entry_path(section, identifier), site=plan.target)
+                result.removed += 1
+            except library_api.LibraryError as error:
+                result.errors.append(f"remove {identifier}: {error}")
+
+    for section, entries in plan.entries.items():
+        on_progress(f"Pushing {len(entries)} {SECTIONS[section]['label']}...")
+        for entry in entries:
+            try:
+                library_api.send("PUT", entry_path(section, entry["id"]), entry["fields"], site=plan.target)
+                result.pushed += 1
+            except library_api.LibraryError as error:
+                result.errors.append(f"{entry['id']}: {error}")
+    return result
+
+
+def main() -> int:
+    try:
+        plan = plan_push()
+    except (OSError, ValueError) as error:
+        print(f"Push stopped: {error}", file=sys.stderr)
+        return 1
+    print(plan.describe())
+    if input("Type yes to continue: ").strip().lower() != "yes":
+        print("Nothing was pushed")
+        return 0
+    result = run_push(plan, print)
+    for message in result.errors:
+        print(f"Error: {message}", file=sys.stderr)
+    print(f"Pushed {result.pushed}, removed {result.removed}, {len(result.errors)} failed")
+    return 1 if result.errors else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/content/tui/dialogs.py

@@ -86,15 +86,17 @@ class FormDialog(ModalScreen[dict | None]):
 class ConfirmDialog(ModalScreen[bool]):
     BINDINGS = [("y", "confirm", "Yes"), ("n", "cancel", "No"), ("escape", "cancel", "No")]
 
-    def __init__(self, message: str):
+    def __init__(self, message: str, confirm_label: str = "Delete", confirm_variant: str = "error"):
         super().__init__()
         self.message = message
+        self.confirm_label = confirm_label
+        self.confirm_variant = confirm_variant
 
     def compose(self) -> ComposeResult:
         with Vertical(classes="dialog"):
             yield Label(self.message, classes="dialog-title")
             with Horizontal(classes="dialog-buttons"):
-                yield Button("Delete", variant="error", id="confirm")
+                yield Button(self.confirm_label, variant=self.confirm_variant, id="confirm")
                 yield Button("Cancel", id="cancel")
 
     @on(Button.Pressed, "#confirm")