From aed0db3cdac02116408f96f587a51d5f69f0a578 Mon Sep 17 00:00:00 2001 From: Worph Date: Sat, 12 Sep 2026 16:37:22 +0200 Subject: [PATCH] improved search --- .github/workflows/publish.yml | 4 +- Cargo.lock | 2 +- Cargo.toml | 6 +- README.md | 60 +- feeder-plugin/book-feeder/Cargo.toml | 13 +- feeder-plugin/book-feeder/Dockerfile | 14 +- feeder-plugin/book-feeder/src/gutenberg.rs | 515 ++++++++- feeder-plugin/book-feeder/src/lib.rs | 14 +- feeder-plugin/book-feeder/src/main.rs | 31 +- feeder-plugin/book-feeder/src/openlibrary.rs | 1010 +++++++++++++++++ .../book-feeder/src/openlibrary_client.rs | 815 +++++++++++++ feeder-plugin/book-feeder/tests/contract.rs | 186 ++- 12 files changed, 2598 insertions(+), 72 deletions(-) create mode 100644 feeder-plugin/book-feeder/src/openlibrary.rs create mode 100644 feeder-plugin/book-feeder/src/openlibrary_client.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 43ac688..aaa6e60 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,8 +3,8 @@ name: Publish image # Build and push the feeder image to the GitHub Container Registry (ghcr.io): # * every push to main -> the moving `main` tag # * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) -# The build context is the repo root (the workspace) so the vendored -# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# The build context is the repo root (the workspace); `meta-feeder-sdk` is a +# tag-pinned git dependency fetched during the build. The Dockerfile lives under # feeder-plugin/book-feeder/. on: push: diff --git a/Cargo.lock b/Cargo.lock index b1b88de..5367d00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -792,6 +792,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "meta-feeder-sdk" version = "0.1.0" +source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.2.1#60c4e855a56fbcbf873bbf0e37a434f27a1ec499" dependencies = [ "anyhow", "async-trait", @@ -806,7 +807,6 @@ dependencies = [ "serde", "serde_json", "sha2", - "tempfile", "thiserror 1.0.69", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 3a8de4b..00f7fbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,10 @@ [workspace] resolver = "2" -members = ["crates/*", "feeder-plugin/*"] +members = ["feeder-plugin/*"] +# The vendored SDK copy is no longer built: book-feeder pins meta-feeder-sdk by +# git tag. Excluded (not deleted) so a stray `cargo` run inside it does not try +# to join this workspace. +exclude = ["crates/meta-feeder-sdk"] [workspace.package] edition = "2021" diff --git a/README.md b/README.md index 3e934a5..7728ec5 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,23 @@ # meta-feeder-book -Book **feeder sidecar** for [MetaMesh](https://github.com/worph/meta-gateway) — -surfaces public-domain books from **Project Gutenberg** (via the -[Gutendex](https://gutendex.com) API) into a meta-gateway. +Book **feeder sidecar** for [MetaMesh](https://github.com/worph/meta-gateway). +One binary, `book-feeder`, hosts two upstreams: -It is the proof-of-concept feeder: a single upstream, no preview-seeding, no -BitTorrent — just enough to exercise the `meta-feeder-sdk` + feeder HTTP -contract end to end. +| Upstream | Serves | What it is | +|----------|--------|------------| +| `gutenberg` | `document` × `book` | Public-domain **editions** from Project Gutenberg (via the [Gutendex](https://gutendex.com) API), with an epub url-locator per record | +| `openlibrary` | `card` × `book` | **Work cards** from [Open Library](https://openlibrary.org) — the book identity tier (`TMDB : video :: AniList : comics :: Open Library : books`) | + +The two meet through `openlibraryid`. A Gutenberg edition carries it only when +Open Library itself records the link: exactly one Open Library work lists the +etext's Project Gutenberg id (`search.json?q=id_project_gutenberg:`). The +edition then also carries the transient trust claim `anchored=true`, +`anchorSource=openlibrary`, which the gateway stores as +`anchoredBy/openlibrary:OL…W`. There is no title or author matching. + +An Open Library card is addressed `compute_card_cid("openlibrary", "OL…W")`, +the bare work id, byte-identical to meta-read's `card_cid::openlibrary`. A work +with no cover or no description gets no card. ## Role in MetaMesh @@ -32,9 +43,16 @@ drives the contract: |---------|---------|-------| | `META_FEEDER_HTTP_LISTEN` | `0.0.0.0:8080` | HTTP listen address | | `META_FEEDER_STATE_DIR` | `/data/meta-feeder` | redb cache + state | +| `OPENLIBRARY_CONTACT` | — | e-mail or URL sent in the Open Library User-Agent. First-boot seed for the `contact` config field; a value saved in the config UI wins | +| `OPENLIBRARY_TOP_N` | `12` | cards per search | +| `OPENLIBRARY_DISCOVERY_N` | `20` | cards per browse row (`popular:true` → Open Library trending, `fresh:true` → recently added) | | `RUST_LOG` | `info` | tracing filter | -No upstream API key is required — Gutendex is open. +No upstream API key is required. Open Library asks API clients to identify +themselves, so set a contact in production: an identified client gets a larger +request budget. Both upstreams share one Open Library rate budget. The config +page edits the `gutenberg` upstream, and the contact saved there is read by +both. ## Image @@ -47,24 +65,32 @@ tag) and on `v*` tags (semver tags). ## Build locally -The build context is the **repo root** (the Cargo workspace) so the vendored -`meta-feeder-sdk` path dependency resolves: +The build context is the **repo root** (the Cargo workspace). +`meta-feeder-sdk` is a git dependency pinned by tag, so the builder needs +network access to github.com: ```bash docker build -f feeder-plugin/book-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-book:dev . ``` +Tests (`cargo` is not needed on the host; wiremock stands in for every upstream): + +```bash +docker run --rm -v "$PWD":/src -w /src -e CARGO_TARGET_DIR=/target rust:1.89 \ + cargo test -p book-feeder +``` + ## Repo layout -This repo is a self-contained Cargo workspace vendored out of the -`meta-gateway` monorepo: - ``` -Cargo.toml # workspace: members = crates/*, feeder-plugin/* -crates/meta-feeder-sdk/ # vendored shared feeder SDK +Cargo.toml # workspace: members = feeder-plugin/* feeder-plugin/book-feeder/ # this feeder's crate + Dockerfile + src/gutenberg.rs # `gutenberg` upstream + src/openlibrary.rs # `openlibrary` upstream (cards) + src/openlibrary_client.rs # Open Library client (budget, cache, binding lookup) +crates/meta-feeder-sdk/ # old vendored SDK copy — no longer built (excluded) ``` -Upstream source of truth for the SDK and the feeder crate is -[`worph/meta-gateway`](https://github.com/worph/meta-gateway); changes there are -vendored back into this repo. +`meta-feeder-sdk` comes from [`worph/meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk) +at the tag in `feeder-plugin/book-feeder/Cargo.toml`. An SDK change reaches +this feeder only when it is tagged and the pin is bumped. diff --git a/feeder-plugin/book-feeder/Cargo.toml b/feeder-plugin/book-feeder/Cargo.toml index dc91c99..49e43cb 100644 --- a/feeder-plugin/book-feeder/Cargo.toml +++ b/feeder-plugin/book-feeder/Cargo.toml @@ -6,9 +6,10 @@ rust-version.workspace = true license.workspace = true publish = false -# Book feeder sidecar: Project Gutenberg (via Gutendex). The proof-of-concept -# feeder — single upstream, no preview-seeding, no BitTorrent — that validates -# the meta-feeder-sdk + feeder HTTP contract end to end. +# Book feeder sidecar, two upstreams: +# - `gutenberg` — Project Gutenberg editions (via Gutendex), the byte tier. +# - `openlibrary` — Open Library work cards, the book identity tier +# (`TMDB : video :: AniList : comics :: Open Library : books`). [lib] name = "book_feeder" @@ -19,7 +20,11 @@ name = "book-feeder" path = "src/main.rs" [dependencies] -meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" } +# Pinned by TAG, never by path. A path dep resolves OUTSIDE the crate, and the +# vendored copy this used to point at had already drifted behind the shared SDK +# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same +# way; see meta-feeder-anilist's manifest. +meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.2.1" } tokio = { version = "1.40", features = ["full"] } axum = { version = "0.7", features = ["json"] } diff --git a/feeder-plugin/book-feeder/Dockerfile b/feeder-plugin/book-feeder/Dockerfile index c2bfdd1..dec4a94 100644 --- a/feeder-plugin/book-feeder/Dockerfile +++ b/feeder-plugin/book-feeder/Dockerfile @@ -1,12 +1,13 @@ # syntax=docker/dockerfile:1 # ============================================================================= -# book-feeder — Project Gutenberg feeder sidecar. Multi-stage build. +# book-feeder — Project Gutenberg + Open Library feeder sidecar. Multi-stage +# build; build context is this repo's root (the Cargo workspace). # -# The build context must be `packages/meta-gateway/` (the workspace root) so -# the `meta-feeder-sdk` path dependency (../../crates/meta-feeder-sdk) resolves. -# Same BuildKit cache-mount strategy as the gateway Dockerfile: the cargo -# registry/git and `target/` persist across builds. The release binary is -# copied OUT of the target cache mount within the same RUN. +# meta-feeder-sdk arrives as a git dependency (tag-pinned), so the vendored +# `crates/` copy is not copied in — `git` must be present in the builder for +# cargo to fetch it. Same BuildKit cache-mount strategy as the gateway +# Dockerfile: the cargo registry/git and `target/` persist across builds. The +# release binary is copied OUT of the target cache mount within the same RUN. # ============================================================================= FROM rust:1.89-slim-bookworm AS builder @@ -20,7 +21,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ -COPY crates/ crates/ COPY feeder-plugin/ feeder-plugin/ RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ diff --git a/feeder-plugin/book-feeder/src/gutenberg.rs b/feeder-plugin/book-feeder/src/gutenberg.rs index a6866e1..2201b94 100644 --- a/feeder-plugin/book-feeder/src/gutenberg.rs +++ b/feeder-plugin/book-feeder/src/gutenberg.rs @@ -6,17 +6,24 @@ //! don't expose an epub variant fail `compute_outcomes` with a permanent //! error (operator-visible; very rare in practice). //! -//! No auth, no API key, no rate-limit configuration. Moved verbatim from the -//! gateway crate's `plugins/gutenberg.rs`; only the import paths changed -//! (`crate::*`/`super::common` → `meta_feeder_sdk::*`, `GatewayPlugin` → -//! `FeederPlugin`). +//! - Per-edition metadata: `GET {rdf_base}/cache/epub/{id}/pg{id}.rdf` for the +//! fields Gutendex does not carry (summary, release date, credits). +//! - Work binding: Open Library's `search.json?q=id_project_gutenberg:{id}` — +//! see [`OpenLibraryClient::gutenberg_work`] and reading-model §2.4. +//! +//! No auth, no API key. Originally moved verbatim from the gateway crate's +//! `plugins/gutenberg.rs`. use std::collections::{BTreeMap, HashMap}; use std::path::Path; +use std::sync::Arc; use async_trait::async_trait; +use meta_feeder_sdk::budget::RateBudget; use meta_feeder_sdk::cache::MidhashCache; use meta_feeder_sdk::common; +use meta_feeder_sdk::config::ConfigSchema; +use meta_feeder_sdk::lang::normalize_lang_code; #[cfg(test)] use meta_feeder_sdk::plugin::HashKind; use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, HashOutcome}; @@ -25,10 +32,22 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth}; use serde::Deserialize; use tracing::{debug, warn}; +use crate::openlibrary_client::{self, OpenLibraryClient}; + /// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`] /// for tests + private mirrors. const DEFAULT_BASE_URL: &str = "https://gutendex.com"; +/// Canonical host for the per-edition RDF. Overridable via +/// [`GutenbergPlugin::with_rdf_base_url`] — until it was, every unit test that +/// ran a query also fetched live RDF from gutenberg.org. +const DEFAULT_RDF_BASE_URL: &str = "https://www.gutenberg.org"; + +/// Upper bound on one binding lookup on the live search path, on top of the +/// budget deadline. A slow Open Library must not hold a Gutenberg result that +/// is already in hand. +const BINDING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(8); + /// Format we treat as the canonical hashable bytes for a Gutenberg record. const EPUB_FORMAT: &str = "application/epub+zip"; @@ -64,7 +83,18 @@ const RETRY_BACKOFF_MS: u64 = 250; pub struct GutenbergPlugin { http: reqwest::Client, base_url: String, + rdf_base_url: String, cache: Option, + /// The Open Library binding client, built at `configure()`. `None` before + /// that — records are then simply unbound. + openlibrary: Option, + /// Test hook for the Open Library API base. Never set in production. + openlibrary_base: Option, + /// Shared with the `openlibrary` plugin by `main.rs`; built at + /// `configure()` when absent. + openlibrary_budget: Option>, + /// The Open Library contact (see `openlibrary_client::load_contact`). + contact: Option, } impl GutenbergPlugin { @@ -80,10 +110,33 @@ impl GutenbergPlugin { Self { http, base_url, + rdf_base_url: DEFAULT_RDF_BASE_URL.to_string(), cache: None, + openlibrary: None, + openlibrary_base: None, + openlibrary_budget: None, + contact: openlibrary_client::env_contact(), } } + /// Fetch the per-edition RDF from `rdf_base_url` instead of gutenberg.org. + pub fn with_rdf_base_url(mut self, rdf_base_url: String) -> Self { + self.rdf_base_url = rdf_base_url; + self + } + + /// Send the Open Library binding lookup to `base` — tests only. + pub fn with_openlibrary_base_url(mut self, base: String) -> Self { + self.openlibrary_base = Some(base); + self + } + + /// Draw binding lookups from a budget shared with the `openlibrary` plugin. + pub fn with_openlibrary_budget(mut self, budget: Arc) -> Self { + self.openlibrary_budget = Some(budget); + self + } + fn cache(&self) -> Result<&MidhashCache, GatewayError> { common::require_cache(self.cache.as_ref(), "gutenberg") } @@ -151,11 +204,13 @@ impl GutenbergPlugin { const BIBREC_CONCURRENCY: usize = 6; let cache = self.cache.clone(); let http = self.http.clone(); + let rdf_base = self.rdf_base_url.trim_end_matches('/').to_string(); let ids: Vec = books.iter().map(|b| b.id).collect(); stream::iter(ids.into_iter()) .map(|id| { let cache = cache.clone(); let http = http.clone(); + let rdf_base = rdf_base.clone(); async move { let record_id = id.to_string(); if let Some(cached) = cache @@ -164,7 +219,7 @@ impl GutenbergPlugin { { return cached; } - let url = format!("https://www.gutenberg.org/cache/epub/{id}/pg{id}.rdf"); + let url = format!("{rdf_base}/cache/epub/{id}/pg{id}.rdf"); let resp = tokio::time::timeout( std::time::Duration::from_secs(8), http.get(&url).send(), @@ -334,6 +389,57 @@ impl GutenbergPlugin { .collect() .await } + + /// Each result's Open Library work, through the Project Gutenberg id Open + /// Library records (reading-model §2.4). Aligned with the input slice; + /// `None` means no binding — none recorded, ambiguous, not configured, over + /// budget, or the lookup failed. + /// + /// ⚠ **A failure never blocks or drops a record.** The binding upgrades an + /// edition that is already readable. Every failure is left uncached, so the + /// next query that surfaces the book tries again; the budget deadline is + /// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the + /// first handful of books and the cache fills the rest over later queries. + async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec> { + use futures::stream::{self, StreamExt}; + const BINDING_CONCURRENCY: usize = 4; + let Some(client) = self.openlibrary.as_ref() else { + return vec![None; books.len()]; + }; + // Owned ids, not `books.iter().map(..)`: a borrowing closure inside the + // stream makes `handle_query`'s future fail the `async_trait` lifetime + // check ("FnOnce is not general enough"). + let ids: Vec = books.iter().map(|b| b.id).collect(); + stream::iter(ids) + .map(|id| async move { + let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE); + match tokio::time::timeout(BINDING_TIMEOUT, lookup).await { + Ok(Ok(work)) => work, + Ok(Err(e)) => { + debug!( + target: "meta-share::gateway", + upstream = "gutenberg", + id, + error = %e, + "open library binding lookup failed; record left unbound" + ); + None + } + Err(_) => { + debug!( + target: "meta-share::gateway", + upstream = "gutenberg", + id, + "open library binding lookup timed out; record left unbound" + ); + None + } + } + }) + .buffered(BINDING_CONCURRENCY) + .collect() + .await + } } impl Default for GutenbergPlugin { @@ -349,7 +455,22 @@ impl FeederPlugin for GutenbergPlugin { } fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { - self.cache = Some(common::open_midhash_cache(cache_dir, "gutenberg")?); + let cache = common::open_midhash_cache(cache_dir, "gutenberg")?; + // The binding cache rides this plugin's own redb (the `misc` table), so + // a binding survives a restart without touching the card plugin's store. + let contact = openlibrary_client::load_contact(cache_dir); + let budget = self + .openlibrary_budget + .clone() + .unwrap_or_else(|| openlibrary_client::budget(contact.is_some())); + self.openlibrary = Some(OpenLibraryClient::new( + budget, + Some(cache.clone()), + self.openlibrary_base.clone(), + contact.as_deref(), + )); + self.contact = contact; + self.cache = Some(cache); Ok(()) } @@ -405,13 +526,19 @@ impl FeederPlugin for GutenbergPlugin { "search returned" ); let raw: Vec = body.results.into_iter().take(max_results).collect(); - let (cover_cids, bibrecs) = - tokio::join!(self.fetch_cover_cids(&raw), self.fetch_bibrec_fields(&raw),); + let (cover_cids, bibrecs, works) = tokio::join!( + self.fetch_cover_cids(&raw), + self.fetch_bibrec_fields(&raw), + self.fetch_openlibrary_works(&raw), + ); Ok(raw .into_iter() .zip(cover_cids) .zip(bibrecs) - .map(|((book, cover_cid), bibrec)| into_discovery_record(book, cover_cid, bibrec)) + .zip(works) + .map(|(((book, cover_cid), bibrec), work)| { + into_discovery_record(book, cover_cid, bibrec, work) + }) .collect()) } @@ -438,7 +565,26 @@ impl FeederPlugin for GutenbergPlugin { .ok() .flatten() .unwrap_or_default(); - let record = into_discovery_record(book, cover_cid, bibrec); + // Not user-facing, so the binding lookup may queue for the full search + // deadline. A failure still leaves the record unbound, never fails the + // compute. + let work = match self.openlibrary.as_ref() { + Some(ol) => ol + .gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE) + .await + .unwrap_or_else(|e| { + debug!( + target: "meta-share::gateway", + upstream = "gutenberg", + record_id, + error = %e, + "open library binding lookup failed; record left unbound" + ); + None + }), + None => None, + }; + let record = into_discovery_record(book, cover_cid, bibrec, work); Ok(common::single_outcome( cid, @@ -481,13 +627,30 @@ impl FeederPlugin for GutenbergPlugin { fn served_content_kinds(&self) -> &'static [&'static str] { &["book"] } + + /// ⚠ Declared HERE, not only on `openlibrary`. The SDK's `/config` page + /// edits the lowest hosted upstream id, which is this one — so this is where + /// the operator sets the contact both upstreams send to Open Library. + fn config_schema(&self) -> ConfigSchema { + ConfigSchema { + fields: vec![openlibrary_client::contact_field()], + } + } + + fn config_values(&self) -> serde_json::Value { + serde_json::json!({ "contact": self.contact.clone().unwrap_or_default() }) + } } /// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`. +/// +/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]: +/// `Some` only when Open Library records exactly one work for this etext. fn into_discovery_record( book: GutendexBook, cover_cid: Option, bibrec: BTreeMap, + openlibrary_work: Option, ) -> DiscoveryRecord { let GutendexBook { id, @@ -520,6 +683,8 @@ fn into_discovery_record( fields.insert(upstream_id_field("gutenberg"), id.to_string()); if !authors.is_empty() { + // Legacy scalar, kept verbatim for existing readers: Gutenberg's own + // "Last, First" spelling, comma-joined. fields.insert( "author".into(), authors @@ -529,9 +694,28 @@ fn into_discovery_record( .join(", "), ); } + // `authors/{name}` key-set (reading-model §3.2): one member per person, in + // display order, so one author compares equal across sources — Open Library + // and AniList both write "Jane Austen", never "Austen, Jane". + for a in &authors { + if let Some(name) = display_name(&a.name) { + fields.insert(format!("authors/{name}"), "true".into()); + } + } if let Some(lang) = languages.first() { fields.insert("language".into(), iso639_1_to_2(lang).to_string()); } + // `languages/{lang3}` key-set for EVERY language, in the stack's shared + // vocabulary (`fre`, where the legacy scalar above says `fra`). Two-letter + // codes only the local table knows (`la`, `ca`, `is`) are widened first; + // anything still not three letters is dropped rather than written as a + // member no `languages:` filter can match. + for lang in &languages { + let code = normalize_lang_code(iso639_1_to_2(&lang.trim().to_ascii_lowercase())); + if code.len() == 3 && code.bytes().all(|b| b.is_ascii_lowercase()) { + fields.insert(format!("languages/{code}"), "true".into()); + } + } if !subjects.is_empty() { fields.insert("subjects".into(), subjects.join(", ")); } @@ -619,10 +803,37 @@ fn into_discovery_record( let _ = download_count; fields.insert("fileName".into(), format!("gutenberg-{id}.epub")); + // The work binding and its trust claim (reading-model §2.4). Written only on + // an identifier link Open Library itself records, so the claim is the plain + // Anchored form: `anchored` + `anchorSource`, and deliberately NO + // `anchorMethod` — that marks a title match, which this never is. The + // gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and + // strips the transient keys before storing. + if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) { + fields.insert("openlibraryid".into(), work); + fields.insert("anchored".into(), "true".into()); + fields.insert("anchorSource".into(), "openlibrary".into()); + } + for (k, v) in bibrec { fields.entry(k).or_insert(v); } + // Registry spellings beside the legacy RDF keys, derived after the merge so + // a bibrec cached before these keys existed produces them too. + // + // `description/eng` from `pgterms:marc520` (kept as `summary`): Gutenberg's + // summaries are written in English whatever the book's language, and + // `description/{lang3}` is the form every card and client reads. + if let Some(summary) = fields.get("summary").cloned() { + fields.entry("description/eng".into()).or_insert(summary); + } + // `releasedate` (iso-date) beside the legacy camelCase `releaseDate`, only + // when the RDF value really is one. + if let Some(date) = fields.get("releaseDate").and_then(|d| iso_date(d)) { + fields.entry("releasedate".into()).or_insert(date); + } + DiscoveryRecord { upstream_id: "gutenberg".into(), record_id, @@ -630,6 +841,38 @@ fn into_discovery_record( } } +/// Gutenberg's "Last, First" → "First Last", the display order every other +/// writer of `authors/{name}` uses. +/// +/// Only when there is **exactly one** comma. "King, Martin Luther, Jr." or a +/// corporate name has no safe reordering, and a wrong one is worse than +/// Gutenberg's own spelling, which is kept as is. +fn display_name(raw: &str) -> Option { + let name = raw.trim(); + if name.is_empty() { + return None; + } + match name.split(',').map(str::trim).collect::>().as_slice() { + [last, first] if !last.is_empty() && !first.is_empty() => Some(format!("{first} {last}")), + _ => Some(name.to_string()), + } +} + +/// `raw` when it is an `iso-date` (`YYYY-MM-DD`, or a bare `YYYY`), else `None`. +/// +/// Never padded: "1998" does not become "1998-01-01", which would claim a +/// precision the RDF never had. +fn iso_date(raw: &str) -> Option { + let digits = |s: &str, n: usize| s.len() == n && s.bytes().all(|b| b.is_ascii_digit()); + let t = raw.trim(); + let ok = match t.split('-').collect::>().as_slice() { + [y] => digits(y, 4), + [y, m, d] => digits(y, 4) && digits(m, 2) && digits(d, 2), + _ => false, + }; + ok.then(|| t.to_string()) +} + /// Parse a Project Gutenberg `pg{id}.rdf` document for per-edition metadata. fn parse_pgterms_rdf(xml: &str) -> BTreeMap { let mut out = BTreeMap::new(); @@ -762,6 +1005,28 @@ mod rdf_parser_tests { let f = parse_pgterms_rdf("not rdf"); assert!(f.is_empty()); } + + /// The legacy RDF keys stay; the registry spellings are derived beside them + /// on the record — including from a bibrec cached before those keys + /// existed, which holds only the legacy ones. + #[test] + fn summary_and_issued_date_also_arrive_under_registry_keys() { + let xml = r#" + 1998-06-01 + A classic novel of manners. + "#; + let book: GutendexBook = serde_json::from_value(serde_json::json!({ + "id": 1342, + "title": "Pride and Prejudice" + })) + .unwrap(); + let r = into_discovery_record(book, None, parse_pgterms_rdf(xml), None); + let get = |k: &str| r.fields.get(k).map(String::as_str); + assert_eq!(get("summary"), Some("A classic novel of manners.")); + assert_eq!(get("description/eng"), Some("A classic novel of manners.")); + assert_eq!(get("releaseDate"), Some("1998-06-01")); + assert_eq!(get("releasedate"), Some("1998-06-01")); + } } /// ISO 639-1 (two-letter) → ISO 639-2/T (three-letter) for the ~20 most common @@ -877,13 +1142,27 @@ mod tests { }) } + /// Every upstream — Gutendex, the RDF host and Open Library — on the one mock + /// server, so no test reaches the internet. An unmounted path 404s, which + /// every optional lookup treats as "nothing there". fn configured_plugin_against(server: &MockServer) -> (GutenbergPlugin, tempfile::TempDir) { let dir = tempfile::tempdir().expect("tempdir"); - let mut plugin = GutenbergPlugin::with_base_url(server.uri()); + let mut plugin = GutenbergPlugin::with_base_url(server.uri()) + .with_rdf_base_url(server.uri()) + .with_openlibrary_base_url(server.uri()) + .with_openlibrary_budget(RateBudget::new(100.0, 100.0)); plugin.configure(dir.path()).expect("configure"); (plugin, dir) } + /// Open Library's answer to `q=id_project_gutenberg:`. + fn binding(id: u64, body: serde_json::Value) -> Mock { + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", format!("id_project_gutenberg:{id}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(body)) + } + /// ⚠ The search path must ask for the CANONICAL `/books/`. /// /// Gutendex 301s `/books` → `/books/`. Following a redirect is free in @@ -1086,7 +1365,16 @@ mod tests { Some("https://www.gutenberg.org/ebooks/1342") ); assert_eq!(r0.fields.get("author").map(String::as_str), Some("Austen, Jane")); + assert_eq!( + r0.fields.get("authors/Jane Austen").map(String::as_str), + Some("true") + ); + assert!(!r0.fields.contains_key("authors/Austen, Jane")); assert_eq!(r0.fields.get("language").map(String::as_str), Some("eng")); + assert_eq!(r0.fields.get("languages/eng").map(String::as_str), Some("true")); + // No Open Library answer mounted: the record is returned, unbound. + assert!(!r0.fields.contains_key("openlibraryid")); + assert!(!r0.fields.contains_key("anchored")); assert_eq!(r0.fields.get("format").map(String::as_str), Some("epub")); assert_eq!( r0.fields.get("fileName").map(String::as_str), @@ -1261,6 +1549,211 @@ mod tests { } } + #[test] + fn display_name_reorders_only_a_single_comma() { + assert_eq!(display_name("Austen, Jane").as_deref(), Some("Jane Austen")); + assert_eq!( + display_name("Shelley, Mary Wollstonecraft").as_deref(), + Some("Mary Wollstonecraft Shelley") + ); + assert_eq!( + display_name("King, Martin Luther, Jr.").as_deref(), + Some("King, Martin Luther, Jr.") + ); + assert_eq!(display_name("Homer").as_deref(), Some("Homer")); + assert_eq!(display_name(" "), None); + } + + #[test] + fn every_language_becomes_a_lang3_key_set_member() { + let book: GutendexBook = serde_json::from_value(serde_json::json!({ + "id": 1, + "title": "Polyglot", + "languages": ["en", "fr", "la", "xx"] + })) + .unwrap(); + let r = into_discovery_record(book, None, BTreeMap::new(), None); + let members: Vec<&str> = r + .fields + .keys() + .filter_map(|k| k.strip_prefix("languages/")) + .collect(); + // The SDK vocabulary (`fre`), every language, and nothing that is not + // three letters. + assert_eq!(members, vec!["eng", "fre", "lat"]); + // The legacy scalar is untouched. + assert_eq!(r.fields.get("language").map(String::as_str), Some("eng")); + } + + #[test] + fn releasedate_is_written_only_for_an_iso_date() { + assert_eq!(iso_date("2008-06-27").as_deref(), Some("2008-06-27")); + assert_eq!(iso_date("1998").as_deref(), Some("1998")); + assert_eq!(iso_date("1998-06"), None); + assert_eq!(iso_date("June 1998"), None); + } + + /// The RDF host is configurable, so the per-edition fields are testable + /// without gutenberg.org. + #[tokio::test] + async fn rdf_summary_and_issue_date_arrive_through_the_configured_rdf_host() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/cache/epub/1342/pg1342.rdf")) + .respond_with(ResponseTemplate::new(200).set_body_string( + r#" + 1998-06-01 + "Pride and Prejudice" is a classic novel. + "#, + )) + .expect(1) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("handle_query"); + let pride = &records[0].fields; + assert!(pride["description/eng"].contains("classic novel")); + assert_eq!(pride.get("releasedate").map(String::as_str), Some("1998-06-01")); + // No RDF mounted for 84: neither key, and the record still arrives. + assert!(!records[1].fields.contains_key("description/eng")); + assert!(!records[1].fields.contains_key("releasedate")); + server.verify().await; + } + + #[tokio::test] + async fn exactly_one_open_library_work_binds_the_edition_with_an_anchored_claim() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + binding( + 1342, + serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671", "45186"] } + ]}), + ) + .mount(&server) + .await; + // Two works list etext 84: an unmerged duplicate, so no binding at all. + binding( + 84, + serde_json::json!({ "numFound": 2, "docs": [ + { "key": "/works/OL450063W", "id_project_gutenberg": ["84"] }, + { "key": "/works/OL18388W", "id_project_gutenberg": ["84"] } + ]}), + ) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("handle_query"); + assert_eq!(records.len(), 2); + + let pride = &records[0].fields; + assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W")); + assert_eq!(pride.get("anchored").map(String::as_str), Some("true")); + assert_eq!(pride.get("anchorSource").map(String::as_str), Some("openlibrary")); + assert!( + !pride.contains_key("anchorMethod"), + "an identifier link Open Library records is not a title match" + ); + + let frankenstein = &records[1].fields; + for k in ["openlibraryid", "anchored", "anchorSource", "anchorMethod"] { + assert!(!frankenstein.contains_key(k), "ambiguous binding must not stamp {k}"); + } + } + + /// ⚠ A binding failure never costs the record. + #[tokio::test] + async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + binding(1342, serde_json::json!({ "numFound": 0, "docs": [] })) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "id_project_gutenberg:84")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("an Open Library failure must not fail the query"); + assert_eq!(records.len(), 2); + for r in &records { + assert!(!r.fields.contains_key("openlibraryid")); + assert!(!r.fields.contains_key("anchored")); + } + } + + /// A book surfaced by every browse must not cost a lookup every browse. + #[tokio::test] + async fn a_binding_answer_is_looked_up_once_per_book() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + binding( + 1342, + serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] } + ]}), + ) + .expect(1) + .mount(&server) + .await; + binding(84, serde_json::json!({ "numFound": 0, "docs": [] })) + .expect(1) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + for _ in 0..2 { + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("handle_query"); + assert_eq!( + records[0].fields.get("openlibraryid").map(String::as_str), + Some("OL66554W") + ); + } + server.verify().await; + } + + /// The SDK's config page shows the lowest upstream id — this one — so the + /// Open Library contact must be settable here. + #[test] + fn the_open_library_contact_is_configurable_on_this_upstream() { + let schema = GutenbergPlugin::new().config_schema(); + assert!(schema.fields.iter().any(|f| f.key == "contact")); + } + #[test] fn health_reflects_configure_state() { let plugin = GutenbergPlugin::new(); diff --git a/feeder-plugin/book-feeder/src/lib.rs b/feeder-plugin/book-feeder/src/lib.rs index 1f4c37e..6c7154f 100644 --- a/feeder-plugin/book-feeder/src/lib.rs +++ b/feeder-plugin/book-feeder/src/lib.rs @@ -1,5 +1,13 @@ -//! `book-feeder` library surface — exposes the Gutenberg plugin so integration -//! tests (and, later, other binaries) can construct it. The binary -//! (`main.rs`) is a thin wrapper over [`meta_feeder_sdk::serve_feeders`]. +//! `book-feeder` library surface — exposes the plugins so integration tests +//! (and, later, other binaries) can construct them. The binary (`main.rs`) is a +//! thin wrapper over [`meta_feeder_sdk::serve_feeders`]. +//! +//! Two upstreams: +//! - [`gutenberg`] — Project Gutenberg editions, the byte tier (`document` × `book`); +//! - [`openlibrary`] — Open Library work cards, the identity tier (`card` × `book`), +//! over the client in [`openlibrary_client`], which `gutenberg` also uses to +//! bind an edition to its work. pub mod gutenberg; +pub mod openlibrary; +pub mod openlibrary_client; diff --git a/feeder-plugin/book-feeder/src/main.rs b/feeder-plugin/book-feeder/src/main.rs index 85c6229..ab91ad8 100644 --- a/feeder-plugin/book-feeder/src/main.rs +++ b/feeder-plugin/book-feeder/src/main.rs @@ -1,17 +1,27 @@ -//! `book-feeder` — Project Gutenberg feeder sidecar. +//! `book-feeder` — the book tier's feeder sidecar. //! -//! Hosts the [`gutenberg::GutenbergPlugin`] over the feeder HTTP contract via -//! [`meta_feeder_sdk::serve_feeders`]. The gateway core reaches it server-to- -//! server on the internal docker network. +//! Hosts two plugins over the feeder HTTP contract via +//! [`meta_feeder_sdk::serve_feeders`]: +//! - [`GutenbergPlugin`] — Project Gutenberg editions (`document` × `book`); +//! - [`OpenLibraryPlugin`] — Open Library work cards (`card` × `book`). +//! +//! The gateway core reaches it server-to-server on the internal docker network. //! //! Env: //! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) //! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `OPENLIBRARY_CONTACT` — e-mail/URL sent in the Open Library User-Agent; +//! first-boot seed for the `contact` config field (the saved field wins) +//! - `OPENLIBRARY_TOP_N`, `OPENLIBRARY_DISCOVERY_N` — optional card result sizing //! - `RUST_LOG` — tracing filter (default `info`) use std::net::SocketAddr; +use std::path::Path; use book_feeder::gutenberg::GutenbergPlugin; +use book_feeder::openlibrary::OpenLibraryPlugin; +use book_feeder::openlibrary_client; +use meta_feeder_sdk::plugin::FeederPlugin; use meta_feeder_sdk::serve_feeders; use tracing_subscriber::EnvFilter; @@ -27,5 +37,16 @@ async fn main() -> anyhow::Result<()> { let state_dir = std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); - serve_feeders(vec![Box::new(GutenbergPlugin::new())], state_dir, listen).await + // ONE budget for both plugins: the Gutenberg → Open Library binding and the + // card plugin call the same upstream, which rate-limits per client, not per + // plugin. Two buckets would each think they had the whole allowance. + let contact = + openlibrary_client::load_contact(&Path::new(&state_dir).join("gateway").join("openlibrary")); + let budget = openlibrary_client::budget(contact.is_some()); + + let plugins: Vec> = vec![ + Box::new(GutenbergPlugin::new().with_openlibrary_budget(budget.clone())), + Box::new(OpenLibraryPlugin::new().with_budget(budget)), + ]; + serve_feeders(plugins, state_dir, listen).await } diff --git a/feeder-plugin/book-feeder/src/openlibrary.rs b/feeder-plugin/book-feeder/src/openlibrary.rs new file mode 100644 index 0000000..a8486a2 --- /dev/null +++ b/feeder-plugin/book-feeder/src/openlibrary.rs @@ -0,0 +1,1010 @@ +//! The `openlibrary` upstream — Open Library work cards, the book identity tier. +//! +//! `TMDB : video :: AniList : comics :: Open Library : books`. It names works and +//! nothing else: no bytes, no editions, no availability. Byte tiers (`gutenberg` +//! in this same binary, torznab and others elsewhere) supply the editions and +//! join them to a card through `openlibraryid`. +//! +//! ## The one contract that must not drift +//! +//! The card address is `compute_card_cid("openlibrary", "OL…W")` — the bare work +//! id, **no kind prefix** — byte-identical to `meta-read/src/card_cid.rs`'s +//! `openlibrary` (reading-model §2.3). Pinned by golden vectors below. Get it +//! wrong and every card lands on a different tile from the one the client +//! routes to. + +use std::collections::{BTreeMap, HashSet}; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use futures::stream::{self, StreamExt}; +use meta_feeder_sdk::budget::RateBudget; +use meta_feeder_sdk::config::ConfigSchema; +use meta_feeder_sdk::hash::compute_card_cid; +use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; +use meta_feeder_sdk::query::GatewayQuery; +use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth}; +use tracing::{debug, warn}; + +use crate::openlibrary_client::{ + self as client, OpenLibraryClient, SearchDoc, BROWSE_DEADLINE, SEARCH_DEADLINE, +}; + +/// This upstream's id, and the card locator's source namespace. +/// +/// ⚠ Cross-repo contract — see the module doc. +pub const UPSTREAM: &str = "openlibrary"; + +/// Default result sizing. `top_n` answers the search box; `discovery_n` fills a +/// curated row. +const DEFAULT_TOP_N: usize = 12; +const DEFAULT_DISCOVERY_N: usize = 20; + +/// Over-fetch factors. The card gate (cover AND description) drops a share of +/// every list — most of a `sort=new` list, which is largely catalogue stubs — +/// and a row that asked for twenty should not render seven. +const SEARCH_OVERFETCH: usize = 2; +const FRESH_OVERFETCH: usize = 4; +const MAX_LIMIT: usize = 100; + +/// Work JSON fetches in flight at once. The rate budget is the real limiter; +/// this only bounds how many leases one row holds while waiting. +const DESCRIPTION_CONCURRENCY: usize = 4; + +/// The card locator for an Open Library work id. `None` for anything that is +/// not a bare `OL…W` work id. +pub fn card_cid(work_id: &str) -> Option { + if !client::is_work_id(work_id) { + return None; + } + compute_card_cid(UPSTREAM, work_id) +} + +fn env_usize(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .filter(|v| *v > 0) + .unwrap_or(default) +} + +/// First non-empty value for `key`. +fn first_filter(q: &GatewayQuery, key: &str) -> Option { + q.filters + .get(key) + .and_then(|v| v.first()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +fn filter_is_true(q: &GatewayQuery, key: &str) -> bool { + q.filters + .get(key) + .map(|v| v.iter().any(|x| x.eq_ignore_ascii_case("true"))) + .unwrap_or(false) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Browse { + Popular, + Fresh, +} + +/// A keyword-less curated row: `popular:true` / `fresh:true` and no free text. +fn browse_kind(q: &GatewayQuery) -> Option { + if !q.free_text.trim().is_empty() { + return None; + } + if filter_is_true(q, "fresh") { + return Some(Browse::Fresh); + } + if filter_is_true(q, "popular") || filter_is_true(q, "trending") { + return Some(Browse::Popular); + } + None +} + +/// The browse markers a card echoes. See [`to_record`]. +const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"]; + +/// Identity and instalment filters this feeder cannot answer. +/// +/// ⚠ An id-anchored query this upstream does not understand must return +/// NOTHING — never fall through to the free-text branch, which would search on +/// the query's title and fill a work page with plausible-looking strangers +/// (measured on the music tier: an `mbReleaseGroupId` query fell through and a +/// BTS album page filled with sixty Grateful Dead recordings). +/// +/// - `gutenbergid` / `isbn` CAN lead to a work (reading-model §2.4), but the +/// binding belongs on the **edition**, where the `gutenberg` upstream writes +/// it. A card answered here carries neither key, so the gateway's +/// `record_matches` would drop it anyway — the only effect of answering is +/// spent budget. +/// - `workcid` names one site's comic; `chapterNumber` & co. scope an +/// instalment. A card is never chapter N (see meta-feeder-anilist's list for +/// the stored-echo incident this guards against). +const FOREIGN_IDENTITY_FILTERS: &[&str] = &[ + "anilistid", + "workcid", + "chapterNumber", + "volumeNumber", + "chapterStart", + "chapterEnd", + "tmdbid", + "imdbid", + "tvdbid", + "gutenbergid", + "isbn", + "olEditionId", + "isrc", + "doi", +]; + +/// The foreign filter on `q`, if any. Every MusicBrainz id (`mbReleaseGroupId`, +/// `mbArtistId`, `mbRecordingId`, …) counts, by prefix, so a new one is +/// declined without a list edit. +fn foreign_identity_filter(q: &GatewayQuery) -> Option<&str> { + q.filters.keys().map(String::as_str).find(|k| { + FOREIGN_IDENTITY_FILTERS.contains(k) + || (k.starts_with("mb") && k[2..].starts_with(|c: char| c.is_ascii_uppercase())) + }) +} + +/// Project one doc plus its work's description into a card record. +/// +/// `None` when the work is not a card: +/// - no work id → no card cid → no `cids/` member, and a record without one is +/// silently dropped by the gateway's store-back while the coverage gate still +/// marks the query covered; +/// - **no cover or no description** — an anchor card promises to describe a +/// work (reading-model §2.5). Open Library holds millions of stub works with +/// neither, and publishing them would fill the Books rows with blank tiles. +/// +/// ⚠ Echoes only the browse markers, not every filter. `record_matches` fails a +/// record that lacks a filter key, so a `popular:true` row needs its cards to +/// carry `popular`; but echoing *all* filters (the AniList tier's approach) +/// would stamp e.g. a `licence:pd` query onto a card — and the gateway stores +/// what passes, permanently. A card that lacks another filter's key does not +/// match it, which is the truth. +pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option { + let id = doc.work_id()?; + let cid = card_cid(id)?; + let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?; + let poster_url = doc.cover_url()?; + let description = description.trim(); + if description.is_empty() { + return None; + } + + let mut fields: BTreeMap = BTreeMap::new(); + + // ── The classification axes ────────────────────────────────────────────── + // + // Whoever writes contentKind writes domain AND workForm in the same breath + // (METADATA_KEYS.md convention 6). A book is a work complete in itself. + fields.insert("fileType".into(), "card".into()); + fields.insert("contentKind".into(), "book".into()); + fields.insert("domain".into(), "literature".into()); + fields.insert("workForm".into(), "standalone".into()); + + // ── Identity ───────────────────────────────────────────────────────────── + // + // ⚠ Exactly ONE cids/ member. `url_key_for` takes the lexicographic .min() + // of them, so a second member would make the store key arbitrary. + fields.insert(format!("cids/{cid}"), "true".into()); + // Bare `OL…W`, the same string that is inside the cid. + fields.insert("openlibraryid".into(), id.to_string()); + fields.insert("title".into(), title.to_string()); + + // `description/{lang3}`, never a flat `description` — the flat key is indexed + // but never written (METADATA_KEYS.md §14.6). + fields.insert("description/eng".into(), description.to_string()); + + let mut seen = HashSet::new(); + let authors: Vec<&str> = doc + .author_name + .iter() + .map(|a| a.trim()) + .filter(|a| !a.is_empty() && seen.insert(*a)) + .collect(); + for a in &authors { + fields.insert(format!("authors/{a}"), "true".into()); + } + if !authors.is_empty() { + fields.insert("author".into(), authors.join(", ")); + } + + // `first_publish_year` is the WORK's year (1813 for Pride and Prejudice), + // which is what a card names. A bare year: the upstream has no finer value. + if let Some(y) = doc.first_publish_year.filter(|y| *y > 0) { + fields.insert("releasedate".into(), format!("{y:04}")); + } + + // ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS + // path fetches it, seeds the bytes and writes the content cid back. + fields.insert("poster_url".into(), poster_url); + + fields.insert( + "sourceUrl".into(), + format!("https://openlibrary.org/works/{id}"), + ); + fields.insert(format!("source/gateway:{UPSTREAM}"), "true".into()); + + for marker in BROWSE_MARKERS { + if filter_is_true(query, marker) { + fields.insert((*marker).to_string(), "true".into()); + } + } + + Some(DiscoveryRecord { + upstream_id: UPSTREAM.into(), + record_id: id.to_string(), + fields, + }) +} + +/// Turn docs into at most `n` cards, fetching each candidate's description. +/// +/// Candidates without a work id, a title or a cover are dropped BEFORE any work +/// JSON is fetched — each fetch spends budget, and those docs cannot become +/// cards whatever the description says. The stream is lazy, so once `n` cards +/// exist no further descriptions are requested. +async fn project( + client: &OpenLibraryClient, + docs: Vec, + query: &GatewayQuery, + n: usize, + deadline: Duration, +) -> Vec { + let mut seen = HashSet::new(); + let candidates: Vec = docs + .into_iter() + .filter(|d| { + d.work_id().is_some() + && d.cover_url().is_some() + && d.title.as_deref().is_some_and(|t| !t.trim().is_empty()) + }) + .filter(|d| seen.insert(d.key.clone())) + .collect(); + + stream::iter(candidates) + .map(|doc| async move { + let id = doc.work_id().unwrap_or_default().to_string(); + let text = client.work_text(&id, deadline).await; + (doc, text) + }) + .buffered(DESCRIPTION_CONCURRENCY) + .filter_map(|(doc, text)| async move { + match text { + Ok(Some(w)) => { + // ⚠ A merged-away id. The index still lists it, but minting a + // card for it would be a second tile for a work that lives + // under another id — which the index usually lists too. + if Some(w.id.as_str()) != doc.work_id() { + debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped"); + return None; + } + to_record(&doc, w.description.as_deref()?, query) + } + Ok(None) => None, + Err(e) => { + debug!(key = doc.key, error = ?e, "openlibrary: work lookup failed; no card"); + None + } + } + }) + .take(n) + .collect() + .await +} + +pub struct OpenLibraryPlugin { + client: Option, + top_n: usize, + discovery_n: usize, + contact: Option, + /// Shared with the `gutenberg` plugin by `main.rs`: both call the same + /// upstream, so they draw on one bucket. + budget: Option>, + /// Test hook: point the client at a mock server. Never set in production. + api_base: Option, +} + +impl Default for OpenLibraryPlugin { + fn default() -> Self { + Self::new() + } +} + +impl OpenLibraryPlugin { + pub fn new() -> Self { + Self { + client: None, + top_n: env_usize("OPENLIBRARY_TOP_N", DEFAULT_TOP_N), + discovery_n: env_usize("OPENLIBRARY_DISCOVERY_N", DEFAULT_DISCOVERY_N), + contact: client::env_contact(), + budget: None, + api_base: None, + } + } + + /// Construct against a specific API base — tests only. + pub fn with_api_base(api_base: impl Into) -> Self { + Self { + api_base: Some(api_base.into()), + ..Self::new() + } + } + + /// Draw on a budget shared with the other Open Library caller in this + /// process instead of building one at `configure()`. + pub fn with_budget(mut self, budget: Arc) -> Self { + self.budget = Some(budget); + self + } +} + +#[async_trait] +impl FeederPlugin for OpenLibraryPlugin { + fn upstream_id(&self) -> &'static str { + UPSTREAM + } + + /// ⚠ `card` only, and `book` only. + /// + /// This is the identity tier: it names works, it does not serve bytes. A + /// `document` axis here would route edition queries — the ones that ask for + /// an actual epub — to an upstream that has none. + fn served_file_types(&self) -> &'static [&'static str] { + &["card"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &["book"] + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + let cache = meta_feeder_sdk::common::open_midhash_cache(cache_dir, UPSTREAM)?; + let contact = client::load_contact(cache_dir); + let budget = self + .budget + .clone() + .unwrap_or_else(|| client::budget(contact.is_some())); + self.client = Some(OpenLibraryClient::new( + budget, + Some(cache), + self.api_base.clone(), + contact.as_deref(), + )); + self.contact = contact; + Ok(()) + } + + fn health(&self) -> PluginHealth { + if self.client.is_some() { + PluginHealth::Ok + } else { + PluginHealth::Degraded { + reason: "not configured".into(), + } + } + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + // Layer A — the routing gate, before any network. + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + // A browse row rejected here is an empty wall with no error anywhere + // — the operator's only clue. Say it loudly. + if browse_kind(query).is_some() { + warn!( + filters = ?query.filters.keys().collect::>(), + "openlibrary: a browse row was routed here but does not match \ + served axes (card × book); the row will render empty" + ); + } + return Ok(Vec::new()); + } + + let Some(client) = self.client.as_ref() else { + return Ok(Vec::new()); + }; + + // A foreign identity anchor: decline rather than degrade. + if let Some(k) = foreign_identity_filter(query) { + debug!(filter = k, "openlibrary: foreign identity anchor; returning nothing"); + return Ok(Vec::new()); + } + + let cap = max_results.max(1); + + // Our own anchor: resolve exactly one card. An id that is not a work id + // answers nothing — it must not fall through to the free-text branch. + if let Some(raw) = first_filter(query, "openlibraryid") { + let id = raw.strip_prefix("/works/").unwrap_or(&raw); + if !client::is_work_id(id) { + return Ok(Vec::new()); + } + let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else { + return Ok(Vec::new()); + }; + return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE).await); + } + + // A curated row. + if let Some(kind) = browse_kind(query) { + let n = self.discovery_n.min(cap); + let docs = match kind { + Browse::Popular => { + client + .trending((n * SEARCH_OVERFETCH).min(MAX_LIMIT), BROWSE_DEADLINE) + .await? + } + Browse::Fresh => { + client + .fresh((n * FRESH_OVERFETCH).min(MAX_LIMIT), BROWSE_DEADLINE) + .await? + } + }; + return Ok(project(client, docs, query, n, BROWSE_DEADLINE).await); + } + + // Free text. + let text = query.free_text.trim(); + if text.is_empty() { + // ⚠ Never `free_text_or_star()` here: `search.json` rejects `q=*` + // with a 422, and a filters-only query is not a request for the + // whole catalogue anyway. + return Ok(Vec::new()); + } + let n = self.top_n.min(cap); + let docs = client + .search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE) + .await?; + Ok(project(client, docs, query, n, SEARCH_DEADLINE).await) + } + + /// A card has no bytes: the record IS the payload. + /// + /// `CardLocator` + `bytes: None` lands in the dispatcher's metadata-only + /// branch. The cover travels as `poster_url` and the gateway's seed path + /// owns it. + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let id = record_id.trim(); + let cid = card_cid(id).ok_or_else(|| { + GatewayError::Permanent(format!("openlibrary: bad work id {record_id}")) + })?; + + // Re-resolve so the stored record is complete even when compute is + // called cold. Cheap: the client caches. A record that resolved to a + // different work (a merge) is not this cid's record. + let record = match self.client.as_ref() { + Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await { + Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE) + .await + .into_iter() + .find(|r| r.record_id == id), + _ => None, + }, + None => None, + }; + + Ok(vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::CardLocator, + bytes: None, + record, + file_extension: None, + }]) + } + + async fn get_blob(&self, _cid: &str) -> Option> { + None + } + + fn config_schema(&self) -> ConfigSchema { + ConfigSchema { + fields: vec![client::contact_field()], + } + } + + fn config_values(&self) -> serde_json::Value { + serde_json::json!({ "contact": self.contact.clone().unwrap_or_default() }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use wiremock::matchers::{any, method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn q(free: &str, filters: &[(&str, &str)]) -> GatewayQuery { + let mut q = GatewayQuery::from_free_text(free); + for (k, v) in filters { + q.filters.insert((*k).into(), vec![(*v).into()]); + } + q + } + + fn pride() -> SearchDoc { + SearchDoc { + key: "/works/OL66554W".into(), + title: Some("Pride and Prejudice".into()), + author_name: vec!["Jane Austen".into()], + cover_i: Some(14348537), + first_publish_year: Some(1813), + id_project_gutenberg: vec![], + } + } + + fn doc_json(id: &str, title: &str, cover: Option) -> serde_json::Value { + json!({ + "key": format!("/works/{id}"), + "title": title, + "author_name": ["Jane Austen"], + "cover_i": cover, + "first_publish_year": 1813, + "language": ["eng"] + }) + } + + async fn plugin_against(server: &MockServer) -> (OpenLibraryPlugin, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let mut p = OpenLibraryPlugin::with_api_base(server.uri()) + .with_budget(RateBudget::new(100.0, 100.0)); + p.configure(dir.path()).unwrap(); + (p, dir) + } + + /// A CONFIGURED plugin whose API base fails the test if it is ever called. + async fn plugin_that_must_not_call_out() -> (OpenLibraryPlugin, MockServer, tempfile::TempDir) { + let server = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + let (p, dir) = plugin_against(&server).await; + (p, server, dir) + } + + /// ⚠ The cross-repo contract. These literals are what `meta-read`'s + /// `card_cid::openlibrary` produces (OL45804W is pinned there). If this test + /// fails, every card this feeder mints lands on a different tile from the + /// one the client routes to. Do not "fix" it by updating the literal — find + /// out which side moved. + #[test] + fn golden_vectors_match_meta_read() { + assert_eq!( + card_cid("OL45804W").as_deref(), + Some("bagdsaaaubnxxazlonruwe4tboj4u6tbugu4dancx") + ); + assert_eq!( + card_cid("OL66554W").as_deref(), + Some("bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx") + ); + // No kind prefix, and nothing that is not a work id. + assert_eq!(card_cid("OL66554W"), compute_card_cid("openlibrary", "OL66554W")); + assert_eq!(card_cid("OL12710531M"), None); + assert_eq!(card_cid("work:OL66554W"), None); + } + + #[test] + fn a_card_carries_exactly_the_contract_fields() { + let r = to_record(&pride(), "A novel of manners.", &q("pride", &[])).unwrap(); + assert_eq!(r.upstream_id, "openlibrary"); + assert_eq!(r.record_id, "OL66554W"); + let expected: BTreeMap = [ + ("fileType", "card"), + ("contentKind", "book"), + ("domain", "literature"), + ("workForm", "standalone"), + ("cids/bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx", "true"), + ("openlibraryid", "OL66554W"), + ("title", "Pride and Prejudice"), + ("authors/Jane Austen", "true"), + ("author", "Jane Austen"), + ("description/eng", "A novel of manners."), + ("poster_url", "https://covers.openlibrary.org/b/id/14348537-L.jpg"), + ("releasedate", "1813"), + ("sourceUrl", "https://openlibrary.org/works/OL66554W"), + ("source/gateway:openlibrary", "true"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + // Exact equality: this is also the assertion that a card carries no + // trust claim (`anchored`, `anchorSource`) — the card IS the anchor — and + // no drifting counts. + assert_eq!(r.fields, expected); + } + + #[test] + fn several_authors_are_a_key_set_and_a_joined_scalar() { + let mut d = pride(); + d.author_name = vec!["A. Writer".into(), "B. Writer".into(), "A. Writer".into()]; + let r = to_record(&d, "x", &q("", &[])).unwrap(); + assert_eq!(r.fields["authors/A. Writer"], "true"); + assert_eq!(r.fields["authors/B. Writer"], "true"); + assert_eq!(r.fields["author"], "A. Writer, B. Writer"); + } + + #[test] + fn no_card_without_a_cover_or_a_description() { + let mut coverless = pride(); + coverless.cover_i = None; + assert!(to_record(&coverless, "A novel.", &q("", &[])).is_none()); + assert!(to_record(&pride(), " ", &q("", &[])).is_none()); + let mut edition = pride(); + edition.key = "/works/OL12710531M".into(); + assert!(to_record(&edition, "A novel.", &q("", &[])).is_none()); + } + + #[test] + fn echoes_browse_markers_but_no_other_filter() { + let r = to_record( + &pride(), + "A novel.", + &q("", &[("popular", "true"), ("licence", "pd"), ("contentKind", "book")]), + ) + .unwrap(); + assert_eq!(r.fields["popular"], "true"); + assert!(!r.fields.contains_key("licence"), "a card must not borrow a query's licence"); + } + + #[test] + fn serves_the_card_tier_only() { + let p = OpenLibraryPlugin::new(); + assert_eq!(p.served_file_types(), &["card"]); + assert_eq!(p.served_content_kinds(), &["book"]); + } + + #[test] + fn browse_markers_map_to_rows() { + assert_eq!(browse_kind(&q("", &[("popular", "true")])), Some(Browse::Popular)); + assert_eq!(browse_kind(&q("", &[("trending", "true")])), Some(Browse::Popular)); + assert_eq!(browse_kind(&q("", &[("fresh", "true")])), Some(Browse::Fresh)); + assert_eq!(browse_kind(&q("austen", &[("popular", "true")])), None); + assert_eq!(browse_kind(&q("", &[])), None); + } + + #[tokio::test] + async fn unconfigured_answers_nothing_rather_than_erroring() { + let p = OpenLibraryPlugin::new(); + let out = p.handle_query(&q("austen", &[("fileType", "card")]), 5).await.unwrap(); + assert!(out.is_empty()); + } + + /// Layer A: an edition query or a comic query never reaches Open Library. + #[tokio::test] + async fn a_query_for_other_axes_is_declined_before_any_request() { + let (p, server, _dir) = plugin_that_must_not_call_out().await; + for filters in [ + vec![("fileType", "document"), ("contentKind", "book")], + vec![("fileType", "card"), ("contentKind", "comic")], + ] { + let out = p.handle_query(&q("pride", &filters), 5).await.unwrap(); + assert!(out.is_empty(), "{filters:?}"); + } + server.verify().await; + } + + #[tokio::test] + async fn every_foreign_identity_anchor_is_declined_before_any_request() { + let (p, server, _dir) = plugin_that_must_not_call_out().await; + let mut keys: Vec<&str> = FOREIGN_IDENTITY_FILTERS.to_vec(); + keys.extend(["mbReleaseGroupId", "mbArtistId", "mbRecordingId"]); + for key in keys { + let query = q("Pride and Prejudice", &[("contentKind", "book"), (key, "1342")]); + let out = p.handle_query(&query, 5).await.expect("declined, not failed"); + assert!(out.is_empty(), "{key} must not degrade to a free-text search"); + } + server.verify().await; + } + + #[tokio::test] + async fn a_malformed_openlibraryid_answers_nothing_rather_than_searching() { + let (p, server, _dir) = plugin_that_must_not_call_out().await; + let query = q("Pride and Prejudice", &[("openlibraryid", "OL66554M")]); + assert!(p.handle_query(&query, 5).await.unwrap().is_empty()); + server.verify().await; + } + + #[tokio::test] + async fn a_filters_only_query_without_a_browse_marker_answers_nothing() { + let (p, server, _dir) = plugin_that_must_not_call_out().await; + let query = q("", &[("domain", "literature"), ("contentKind", "book"), ("licence", "pd")]); + assert!(p.handle_query(&query, 5).await.unwrap().is_empty()); + server.verify().await; + } + + #[tokio::test] + async fn a_search_mints_one_card_per_work_with_a_cover_and_a_description() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "pride and prejudice")) + .and(query_param("fields", "key,title,author_name,cover_i,first_publish_year,language")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "numFound": 4, + "docs": [ + doc_json("OL66554W", "Pride and Prejudice", Some(14348537)), + // No cover: no card, and no work JSON is fetched for it. + doc_json("OL15165350W", "Pride and Prejudice", None), + // A work with no description: no card. + doc_json("OL1W", "Stub", Some(5)), + // An edition key under /works/: never a card. + { "key": "/works/OL12710531M", "title": "Stub", "cover_i": 7 } + ] + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL66554W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "key": "/works/OL66554W", + "description": "Pride and Prejudice is an 1813 novel.\r\n\r\n----------\r\nAlso contained in: …" + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL1W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL1W", "title": "Stub" }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL15165350W.json")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let query = q("pride and prejudice", &[("fileType", "card")]); + let out = p.handle_query(&query, 10).await.unwrap(); + assert_eq!(out.len(), 1); + let f = &out[0].fields; + assert_eq!(f["openlibraryid"], "OL66554W"); + assert_eq!(f["description/eng"], "Pride and Prejudice is an 1813 novel."); + assert_eq!(f["cids/bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx"], "true"); + + // The repeat is served from the cache: every mock above is `expect(1)`. + let again = p.handle_query(&query, 10).await.unwrap(); + assert_eq!(again.len(), 1); + assert_eq!(again[0].fields, out[0].fields); + server.verify().await; + } + + #[tokio::test] + async fn an_openlibraryid_resolves_exactly_one_card_and_reads_a_typed_description() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "key:/works/OL66554W")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "numFound": 1, + "docs": [doc_json("OL66554W", "Pride and Prejudice", Some(14348537))] + }))) + .mount(&server) + .await; + // The free text must NOT be searched when the anchor is present. + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "Pride and Prejudice")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL66554W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "key": "/works/OL66554W", + "description": { "type": "/type/text", "value": "A novel of manners." } + }))) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let out = p + .handle_query( + &q("Pride and Prejudice", &[("openlibraryid", "OL66554W"), ("contentKind", "book")]), + 10, + ) + .await + .unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].fields["description/eng"], "A novel of manners."); + assert_eq!(out[0].fields["releasedate"], "1813"); + server.verify().await; + } + + /// A work the search index does not hold still resolves, from the work JSON + /// and its author records — with no year, rather than an edition's. + #[tokio::test] + async fn an_unindexed_work_falls_back_to_the_work_json_and_its_authors() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "key:/works/OL45804W")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "numFound": 0, "docs": [] }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL45804W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "key": "/works/OL45804W", + "title": "Fantastic Mr Fox", + "covers": [-1, 6498519], + "authors": [{ "author": { "key": "/authors/OL34184A" }, "type": { "key": "/type/author_role" } }], + "first_publish_date": "1988", + "description": "A clever fox outwits three farmers." + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/authors/OL34184A.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "name": "Roald Dahl" }))) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let out = p + .handle_query(&q("", &[("openlibraryid", "OL45804W")]), 10) + .await + .unwrap(); + assert_eq!(out.len(), 1); + let f = &out[0].fields; + assert_eq!(f["title"], "Fantastic Mr Fox"); + assert_eq!(f["author"], "Roald Dahl"); + assert_eq!(f["poster_url"], "https://covers.openlibrary.org/b/id/6498519-L.jpg"); + assert_eq!(f["cids/bagdsaaaubnxxazlonruwe4tboj4u6tbugu4dancx"], "true"); + assert!(!f.contains_key("releasedate"), "an edition's date is not the work's"); + } + + /// ⚠ A merged work answers 200 with a redirect body. The card must be minted + /// once, under the id the work lives under. + #[tokio::test] + async fn a_merged_work_is_not_minted_under_its_old_id() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "mr fox")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "numFound": 2, + "docs": [ + doc_json("OL45883W", "Fantastic Mr Fox", Some(1)), + doc_json("OL45804W", "Fantastic Mr Fox", Some(6498519)) + ] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL45883W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "key": "/works/OL45883W", + "type": { "key": "/type/redirect" }, + "location": "/works/OL45804W" + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL45804W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "key": "/works/OL45804W", + "description": "A clever fox." + }))) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let out = p.handle_query(&q("mr fox", &[]), 10).await.unwrap(); + let ids: Vec<&str> = out.iter().map(|r| r.record_id.as_str()).collect(); + assert_eq!(ids, vec!["OL45804W"]); + } + + #[tokio::test] + async fn the_popular_row_reads_trending_and_echoes_its_marker() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/trending/daily.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "query": "/trending/daily", + "works": [doc_json("OL66554W", "Pride and Prejudice", Some(14348537))] + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/search.json")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL66554W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL66554W", "description": "A novel." }))) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let row = q("", &[("domain", "literature"), ("contentKind", "book"), ("popular", "true")]); + let out = p.handle_query(&row, 20).await.unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].fields["popular"], "true"); + server.verify().await; + } + + #[tokio::test] + async fn the_fresh_row_sorts_search_by_new() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("sort", "new")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "numFound": 1, + "docs": [doc_json("OL66554W", "Pride and Prejudice", Some(14348537))] + }))) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/works/OL66554W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL66554W", "description": "A novel." }))) + .mount(&server) + .await; + + let (p, _dir) = plugin_against(&server).await; + let out = p.handle_query(&q("", &[("fresh", "true")]), 20).await.unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].fields["fresh"], "true"); + server.verify().await; + } + + #[tokio::test] + async fn a_rate_limited_search_surfaces_as_rate_limited() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "30")) + .mount(&server) + .await; + let (p, _dir) = plugin_against(&server).await; + let err = p.handle_query(&q("austen", &[]), 5).await.expect_err("429"); + assert!( + matches!(err, GatewayError::RateLimited { retry_after_s: 30 }), + "got {err:?}" + ); + } + + #[tokio::test] + async fn compute_outcomes_is_a_byteless_card_locator() { + let p = OpenLibraryPlugin::new(); + let out = p.compute_outcomes("OL66554W").await.unwrap(); + assert_eq!(out.len(), 1); + assert!(out[0].bytes.is_none(), "a card has no bytes"); + assert!(out[0].file_extension.is_none(), "must be None when bytes is None"); + assert_eq!(out[0].hash_kind, HashKind::CardLocator); + assert_eq!(out[0].hash.0, "bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx"); + } + + #[tokio::test] + async fn compute_outcomes_rejects_a_non_work_id() { + let p = OpenLibraryPlugin::new(); + assert!(p.compute_outcomes("OL12710531M").await.is_err()); + assert!(p.compute_outcomes("1342").await.is_err()); + } +} diff --git a/feeder-plugin/book-feeder/src/openlibrary_client.rs b/feeder-plugin/book-feeder/src/openlibrary_client.rs new file mode 100644 index 0000000..fcd7239 --- /dev/null +++ b/feeder-plugin/book-feeder/src/openlibrary_client.rs @@ -0,0 +1,815 @@ +//! The Open Library HTTP client: rate budget, redb read-through cache, and the +//! calls the book tier makes. +//! +//! Shared by both upstreams this binary hosts: +//! - the `openlibrary` card plugin — search, trending, "new", work lookups; +//! - the `gutenberg` plugin — the `id_project_gutenberg` binding lookup +//! (reading-model §2.4), which is what lets an edition join its work. +//! +//! Plain JSON over `reqwest`, like every feeder in the fleet. Shapes confirmed +//! against the live API on 2026-09-11: `search.json` answers `{numFound, docs}`, +//! `/trending/daily.json` answers `{works}` with the same doc shape (and honours +//! `fields=`), and a work JSON's `description` is either a bare string or a +//! `{type: "/type/text", value}` object — both occur in the catalogue. + +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use meta_feeder_sdk::budget::{Lease, RateBudget}; +use meta_feeder_sdk::cache::MidhashCache; +use meta_feeder_sdk::config::ConfigField; +use meta_feeder_sdk::types::GatewayError; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{debug, warn}; + +/// Canonical Open Library base URL. Overridable for tests (wiremock). +pub const DEFAULT_API_BASE: &str = "https://openlibrary.org"; + +/// Cover image base. `-L` is the large rendition; the gateway seeds whatever +/// `poster_url` names, so one size is published, not three. +const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id"; + +/// Config key (and env seed) for the operator contact sent in the User-Agent. +pub const CONTACT_KEY: &str = "contact"; +pub const CONTACT_ENV: &str = "OPENLIBRARY_CONTACT"; + +/// How long a caller waits for a rate-budget lease. +/// +/// Three callers, three patiences. A **search** has someone watching a spinner. +/// A **browse row** is painted by a timer and is worth less late than never +/// blocking the search behind it. A **binding lookup** rides on a Gutenberg +/// result that is already in hand, and must never make that result late: an +/// unbound edition is still a readable edition, and the next query binds it. +pub const SEARCH_DEADLINE: Duration = Duration::from_secs(20); +pub const BROWSE_DEADLINE: Duration = Duration::from_secs(5); +pub const BINDING_DEADLINE: Duration = Duration::from_secs(2); + +/// HTTP timeout for a single upstream call. +const HTTP_TIMEOUT_SECS: u64 = 20; + +/// Attempts per call when the request never produced a response. openlibrary.org +/// refused connections outright for a stretch while this tier was being planned; +/// one retry covers a blink without hammering an upstream that answered. +const SEND_ATTEMPTS: usize = 2; +const RETRY_BACKOFF_MS: u64 = 250; + +/// The field selection for every doc-shaped request, written once so a card +/// resolved by search and the same card resolved by id are byte-identical. +const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language"; + +/// Cache lifetimes. +/// +/// ⚠ Unlike AniList's, these lists are not immutable. `trending/daily` changes +/// every day and `sort=new` every hour; a permanent cache would freeze the +/// Popular Books row on whatever it showed the first time. Work JSON and author +/// names are edited rarely and are cached long. +const DAY: u64 = 86_400; +const WORK_TTL_SECS: u64 = 30 * DAY; +const SEARCH_TTL_SECS: u64 = DAY; +const BROWSE_TTL_SECS: u64 = 6 * 3600; +/// A binding is a fact Open Library records; it is re-checked monthly. A miss is +/// re-checked weekly, because Open Library librarians add Gutenberg ids to +/// editions all the time and a permanent negative would never see them. +const BINDING_HIT_TTL_SECS: u64 = 30 * DAY; +const BINDING_MISS_TTL_SECS: u64 = 7 * DAY; + +/// Authors resolved on the work-JSON fallback. Each is one more request. +const MAX_AUTHORS: usize = 4; + +/// Build the process budget for Open Library. +/// +/// Open Library asks API clients to identify themselves (app name + contact in +/// the User-Agent) and throttles anonymous traffic first, so an identified +/// client gets a larger bucket. The exact published ceiling could not be read +/// back from the developer page when this was written — an overestimate is +/// safe regardless: a 429 freezes the whole bucket for its `Retry-After` via +/// `note_throttled`, rather than each caller discovering it alone. +pub fn budget(identified: bool) -> Arc { + if identified { + RateBudget::new(3.0, 6.0) + } else { + RateBudget::new(1.0, 3.0) + } +} + +/// The operator contact, file-or-env. +/// +/// ⚠ **Looked up in both plugins' config dirs, not just the caller's.** The +/// SDK's `/config` page edits the *lowest* hosted upstream id, which in this +/// binary is `gutenberg`, so a contact saved through the dashboard lands in +/// `gateway/gutenberg/config.json` — yet both upstreams send requests to Open +/// Library and both must identify themselves. File wins over env, as for every +/// feeder config. +pub fn load_contact(cache_dir: &Path) -> Option { + let mut dirs = vec![cache_dir.to_path_buf()]; + if let Some(parent) = cache_dir.parent() { + for id in ["gutenberg", "openlibrary"] { + dirs.push(parent.join(id)); + } + } + for dir in dirs { + let Ok(bytes) = std::fs::read(dir.join("config.json")) else { + continue; + }; + let Ok(v) = serde_json::from_slice::(&bytes) else { + continue; + }; + if let Some(c) = v.get(CONTACT_KEY).and_then(Value::as_str).and_then(sanitize_contact) { + return Some(c); + } + } + env_contact() +} + +/// The first-boot seed: `OPENLIBRARY_CONTACT`. +pub fn env_contact() -> Option { + std::env::var(CONTACT_ENV).ok().as_deref().and_then(sanitize_contact) +} + +/// The config field both plugins declare (see [`load_contact`] for why both). +pub fn contact_field() -> ConfigField { + ConfigField::text(CONTACT_KEY, "Open Library contact").with_help( + "An e-mail address or URL sent in the User-Agent of every Open Library request \ + (the openlibrary cards and the gutenberg → Open Library binding). Open Library \ + asks API clients to identify themselves; an identified client gets a larger \ + request budget.", + ) +} + +/// Reduce a contact to something a header can carry. +/// +/// `build_http_client` treats a client build failure as impossible and panics, +/// and an invalid header value is exactly such a failure — so a stray newline +/// in a config field must not reach it. Parentheses are dropped too: they would +/// close the User-Agent comment early. +fn sanitize_contact(raw: &str) -> Option { + let c: String = raw + .chars() + .filter(|c| c.is_ascii_graphic() || *c == ' ') + .filter(|c| !matches!(c, '(' | ')')) + .take(128) + .collect(); + let c = c.trim(); + (!c.is_empty()).then(|| c.to_string()) +} + +fn user_agent(contact: Option<&str>) -> String { + let base = concat!( + "meta-feeder-book/", + env!("CARGO_PKG_VERSION"), + " (gateway:openlibrary; +https://github.com/worph/meta-feeder-book" + ); + match contact { + Some(c) => format!("{base}; {c})"), + None => format!("{base})"), + } +} + +/// Is `s` a bare Open Library **work** id (`OL66554W`)? +/// +/// ⚠ The `W` is the point. `OL…M` is an edition and `OL…A` an author, and the +/// search index does return edition keys under `/works/` for some junk records +/// (`/works/OL12710531M`, observed live on a `first_publish_year` query) — an +/// edition minted as a card would be a second tile for every printing. +pub fn is_work_id(s: &str) -> bool { + s.len() > 3 + && s.starts_with("OL") + && s.ends_with('W') + && s[2..s.len() - 1].bytes().all(|b| b.is_ascii_digit()) +} + +/// One search/trending doc — the subset this feeder reads. +/// +/// ⚠ Nothing mutable is read: no `edition_count`, `ratings_*`, +/// `want_to_read_count` or `ebook_access`. All drift without the work changing +/// (METADATA_KEYS rule #4). +#[derive(Debug, Clone, Default, Deserialize)] +pub struct SearchDoc { + #[serde(default)] + pub key: String, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub author_name: Vec, + #[serde(default)] + pub cover_i: Option, + #[serde(default)] + pub first_publish_year: Option, + #[serde(default)] + pub id_project_gutenberg: Vec, +} + +impl SearchDoc { + /// The bare work id, when the key names a work. + pub fn work_id(&self) -> Option<&str> { + self.key.strip_prefix("/works/").filter(|id| is_work_id(id)) + } + + /// The large cover URL. Open Library uses `-1` for "no cover" in `covers` + /// arrays, so anything not positive is absent. + pub fn cover_url(&self) -> Option { + self.cover_i + .filter(|c| *c > 0) + .map(|c| format!("{COVERS_BASE}/{c}-L.jpg")) + } +} + +/// What a work JSON contributes to a card. +#[derive(Debug, Clone)] +pub struct WorkText { + /// The id the work JSON actually lives under — differs from the id asked + /// for when that one was merged away (see [`OpenLibraryClient::work_json`]). + pub id: String, + pub description: Option, +} + +/// Cache envelope: the body plus when it was fetched, so lists can expire. +#[derive(Serialize, Deserialize)] +struct Cached { + at: u64, + body: Value, +} + +fn now_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +pub struct OpenLibraryClient { + http: reqwest::Client, + budget: Arc, + cache: Option, + /// Overridable so tests can point the client at a wiremock server. + api_base: String, +} + +impl OpenLibraryClient { + pub fn new( + budget: Arc, + cache: Option, + api_base: Option, + contact: Option<&str>, + ) -> Self { + Self { + http: meta_feeder_sdk::common::build_http_client( + HTTP_TIMEOUT_SECS, + &user_agent(contact), + None, + ), + budget, + cache, + api_base: api_base.unwrap_or_else(|| DEFAULT_API_BASE.to_string()), + } + } + + fn cache_get(&self, key: &str) -> Option { + let hit = self.cache.as_ref()?.get_misc(key).ok().flatten()?; + serde_json::from_str::(&hit).ok() + } + + fn cache_put(&self, key: &str, body: &Value) { + let Some(c) = &self.cache else { return }; + let envelope = Cached { + at: now_secs(), + body: body.clone(), + }; + if let Ok(s) = serde_json::to_string(&envelope) { + if let Err(e) = c.put_misc(key, &s) { + warn!(upstream = "openlibrary", key, error = ?e, "cache put failed (non-fatal)"); + } + } + } + + /// One GET, budget-gated, never cached. + async fn get( + &self, + path: &str, + query: &[(&str, String)], + deadline: Duration, + ) -> Result { + if matches!(self.budget.acquire(deadline).await, Lease::DeadlineExceeded) { + // Transient, not Permanent: this upstream is busy, not broken. + return Err(GatewayError::Transient( + "openlibrary: rate budget deadline exceeded".into(), + )); + } + let url = format!("{}{path}", self.api_base.trim_end_matches('/')); + let mut last = String::new(); + let mut response = None; + for attempt in 1..=SEND_ATTEMPTS { + match self.http.get(&url).query(query).send().await { + Ok(r) => { + response = Some(r); + break; + } + Err(e) => { + last = e.to_string(); + warn!(upstream = "openlibrary", path, attempt, error = %last, "transport failure"); + if attempt < SEND_ATTEMPTS { + tokio::time::sleep(Duration::from_millis(RETRY_BACKOFF_MS)).await; + } + } + } + } + let resp = response.ok_or_else(|| { + GatewayError::Transient(format!("openlibrary: GET {path}: {last}")) + })?; + // 429 carries Retry-After; freeze the whole bucket, not just this call. + if let Err(e) = meta_feeder_sdk::common::map_status(&resp) { + if let GatewayError::RateLimited { retry_after_s } = &e { + self.budget + .note_throttled(Duration::from_secs(*retry_after_s as u64)); + } + return Err(e); + } + resp.json::() + .await + .map_err(|e| GatewayError::Transient(format!("openlibrary: bad json from {path}: {e}"))) + } + + /// [`Self::get`] behind the read-through cache. Only successes are cached: + /// a transport blink or a 5xx must never be remembered as "nothing here". + async fn get_cached( + &self, + key: &str, + ttl_secs: u64, + path: &str, + query: &[(&str, String)], + deadline: Duration, + ) -> Result { + if let Some(c) = self.cache_get(key) { + if now_secs().saturating_sub(c.at) <= ttl_secs { + return Ok(c.body); + } + } + let body = self.get(path, query, deadline).await?; + self.cache_put(key, &body); + Ok(body) + } + + /// Free-text search. + pub async fn search( + &self, + text: &str, + limit: usize, + deadline: Duration, + ) -> Result, GatewayError> { + let normalised = text.split_whitespace().collect::>().join(" ").to_lowercase(); + let body = self + .get_cached( + &format!("openlibrary:search:{limit}:{normalised}"), + SEARCH_TTL_SECS, + "/search.json", + &[ + ("q", text.trim().to_string()), + ("fields", SEARCH_FIELDS.into()), + ("limit", limit.to_string()), + ], + deadline, + ) + .await?; + Ok(parse_docs(body.get("docs"))) + } + + /// The `popular:true` row: what Open Library readers opened today. + /// + /// The ordering is a popularity signal, but no count is read or stored. + pub async fn trending( + &self, + limit: usize, + deadline: Duration, + ) -> Result, GatewayError> { + let body = self + .get_cached( + &format!("openlibrary:trending:daily:{limit}"), + BROWSE_TTL_SECS, + "/trending/daily.json", + &[("fields", SEARCH_FIELDS.into()), ("limit", limit.to_string())], + deadline, + ) + .await?; + Ok(parse_docs(body.get("works"))) + } + + /// The `fresh:true` row: recently added works first published this year or + /// last. + /// + /// ⚠ `search.json` rejects `q=*` (422, "Query too short"), and `sort=new` + /// over a broad query is dominated by coverless catalogue stubs. The year + /// window keeps the row about new books; the card gate (cover + description) + /// drops the stubs that remain, so the caller over-fetches. + pub async fn fresh( + &self, + limit: usize, + deadline: Duration, + ) -> Result, GatewayError> { + let year = 1970 + now_secs() / 31_556_952; + let body = self + .get_cached( + &format!("openlibrary:fresh:{year}:{limit}"), + BROWSE_TTL_SECS, + "/search.json", + &[ + ("q", format!("first_publish_year:[{} TO {year}]", year - 1)), + ("sort", "new".into()), + ("fields", SEARCH_FIELDS.into()), + ("limit", limit.to_string()), + ], + deadline, + ) + .await?; + Ok(parse_docs(body.get("docs"))) + } + + /// Resolve one work id to a doc — the anchored lookup. + /// + /// Search by key first, so the record matches what a search would have + /// produced (same fields, same `first_publish_year`; the work JSON's + /// `first_publish_date` is frequently an edition's date — `1853` for a novel + /// first published in 1813). The work JSON plus its author records are the + /// fallback for a work the search index does not hold; that path carries no + /// year, because absent is better than wrong. + pub async fn doc_by_work_id( + &self, + id: &str, + deadline: Duration, + ) -> Result, GatewayError> { + let body = self + .get_cached( + &format!("openlibrary:key:{id}"), + WORK_TTL_SECS, + "/search.json", + &[ + ("q", format!("key:/works/{id}")), + ("fields", SEARCH_FIELDS.into()), + ("limit", "1".into()), + ], + deadline, + ) + .await?; + if let Some(doc) = parse_docs(body.get("docs")) + .into_iter() + .find(|d| d.work_id() == Some(id)) + { + return Ok(Some(doc)); + } + + let Some((work_id, work)) = self.work_json(id, deadline).await? else { + return Ok(None); + }; + let author_keys: Vec = work + .get("authors") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|x| x.pointer("/author/key").and_then(Value::as_str)) + .filter_map(|k| k.strip_prefix("/authors/")) + .map(str::to_string) + .take(MAX_AUTHORS) + .collect() + }) + .unwrap_or_default(); + let mut author_name = Vec::new(); + for author in author_keys { + match self.author_name(&author, deadline).await { + Ok(Some(name)) => author_name.push(name), + Ok(None) => {} + Err(e) => { + debug!(upstream = "openlibrary", author, error = ?e, "author lookup failed") + } + } + } + Ok(Some(SearchDoc { + key: format!("/works/{work_id}"), + title: work.get("title").and_then(Value::as_str).map(str::to_string), + author_name, + cover_i: work + .get("covers") + .and_then(Value::as_array) + .and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)), + first_publish_year: None, + id_project_gutenberg: Vec::new(), + })) + } + + async fn author_name(&self, id: &str, deadline: Duration) -> Result, GatewayError> { + let body = match self + .get_cached( + &format!("openlibrary:author:{id}"), + WORK_TTL_SECS, + &format!("/authors/{id}.json"), + &[], + deadline, + ) + .await + { + Ok(b) => b, + Err(GatewayError::NotFound) => return Ok(None), + Err(e) => return Err(e), + }; + Ok(body + .get("name") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string)) + } + + /// The work JSON and the id it actually lives under, following one merge + /// redirect. + /// + /// ⚠ A merged work answers **200** with `{"type": {"key": "/type/redirect"}, + /// "location": "/works/OL…W"}` (observed live: `OL45883W` → `OL45804W`) — + /// not an HTTP redirect, so an HTTP client follows nothing and a naive + /// reader finds no description. + async fn work_json( + &self, + id: &str, + deadline: Duration, + ) -> Result, GatewayError> { + let mut id = id.to_string(); + for _ in 0..2 { + let body = match self + .get_cached( + &format!("openlibrary:work:{id}"), + WORK_TTL_SECS, + &format!("/works/{id}.json"), + &[], + deadline, + ) + .await + { + Ok(b) => b, + Err(GatewayError::NotFound) => return Ok(None), + Err(e) => return Err(e), + }; + if body.pointer("/type/key").and_then(Value::as_str) != Some("/type/redirect") { + return Ok(Some((id, body))); + } + match body + .get("location") + .and_then(Value::as_str) + .and_then(|l| l.strip_prefix("/works/")) + .filter(|l| is_work_id(l)) + { + Some(next) => id = next.to_string(), + None => return Ok(None), + } + } + Ok(None) + } + + /// A work's canonical id and plain-text description. + pub async fn work_text(&self, id: &str, deadline: Duration) -> Result, GatewayError> { + Ok(self.work_json(id, deadline).await?.map(|(id, work)| WorkText { + id, + description: text_value(work.get("description")).and_then(|d| clean_description(&d)), + })) + } + + /// The Open Library work that records Project Gutenberg etext `gutenberg_id`. + /// + /// `Some` only when **exactly one** work lists it — an identifier link Open + /// Library itself records, which is what grades a binding Anchored + /// (reading-model §2.4). Zero works is no binding; two or more is an + /// unmerged duplicate in Open Library, and picking one would be a guess. + /// + /// ⚠ Never a title or author search. That is a Matched-grade binding and is + /// out of scope here; emitting it under an anchored claim would promote a + /// resemblance to a proof. + pub async fn gutenberg_work( + &self, + gutenberg_id: u64, + deadline: Duration, + ) -> Result, GatewayError> { + let key = format!("openlibrary:gutenberg:{gutenberg_id}"); + if let Some(c) = self.cache_get(&key) { + let work = c.body.get("work").and_then(Value::as_str).map(str::to_string); + let ttl = if work.is_some() { + BINDING_HIT_TTL_SECS + } else { + BINDING_MISS_TTL_SECS + }; + if now_secs().saturating_sub(c.at) <= ttl { + return Ok(work); + } + } + let body = self + .get( + "/search.json", + &[ + ("q", format!("id_project_gutenberg:{gutenberg_id}")), + ("fields", "key,id_project_gutenberg".into()), + // Two, not one: `limit=1` would make "exactly one" unprovable. + ("limit", "2".into()), + ], + deadline, + ) + .await?; + let work = single_gutenberg_work(&body, gutenberg_id); + self.cache_put(&key, &serde_json::json!({ "work": work })); + Ok(work) + } +} + +/// Decide a binding from a `q=id_project_gutenberg:` response. +fn single_gutenberg_work(body: &Value, gutenberg_id: u64) -> Option { + let docs = parse_docs(body.get("docs")); + let total = body + .get("numFound") + .or_else(|| body.get("num_found")) + .and_then(Value::as_u64) + .unwrap_or(docs.len() as u64); + if total != 1 || docs.len() != 1 { + return None; + } + let doc = &docs[0]; + // The doc must itself list the etext. The query is an exact field match, so + // this only fails on an index quirk — and a binding graded Anchored cannot + // rest on a quirk. + let wanted = gutenberg_id.to_string(); + if !doc.id_project_gutenberg.iter().any(|g| g.trim() == wanted) { + return None; + } + doc.work_id().map(str::to_string) +} + +fn parse_docs(v: Option<&Value>) -> Vec { + v.and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|d| serde_json::from_value::(d.clone()).ok()) + .collect() + }) + .unwrap_or_default() +} + +/// A text field that is either `"…"` or `{"type": "/type/text", "value": "…"}`. +fn text_value(v: Option<&Value>) -> Option { + match v? { + Value::String(s) => Some(s.clone()), + Value::Object(o) => o.get("value").and_then(Value::as_str).map(str::to_string), + _ => None, + } +} + +/// Community descriptions carry Windows line endings and, often, a trailing +/// `----------` rule followed by "Also contained in:" link lists. The rule and +/// what follows are catalogue furniture, not synopsis. +fn clean_description(raw: &str) -> Option { + let text = raw.replace("\r\n", "\n").replace('\r', "\n"); + let body = text.split("\n----").next().unwrap_or_default().trim(); + (!body.is_empty()).then(|| body.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn work_ids_are_works_only() { + assert!(is_work_id("OL66554W")); + assert!(!is_work_id("OL12710531M"), "an edition key is not a work"); + assert!(!is_work_id("OL21594A"), "an author key is not a work"); + assert!(!is_work_id("OLW")); + assert!(!is_work_id("/works/OL66554W")); + } + + #[test] + fn description_reads_both_live_shapes() { + let plain = serde_json::json!({ "description": "A novel." }); + let typed = serde_json::json!({ "description": { "type": "/type/text", "value": "A novel." } }); + assert_eq!(text_value(plain.get("description")).as_deref(), Some("A novel.")); + assert_eq!(text_value(typed.get("description")).as_deref(), Some("A novel.")); + assert_eq!(text_value(serde_json::json!({ "description": 3 }).get("description")), None); + } + + #[test] + fn description_drops_the_link_list_after_the_rule() { + let raw = "First para.\r\n\r\nSecond para.\r\n\r\n----------\r\nAlso contained in:\r\n[x](/works/OL1W)"; + assert_eq!( + clean_description(raw).as_deref(), + Some("First para.\n\nSecond para.") + ); + assert_eq!(clean_description(" \r\n "), None); + } + + #[test] + fn cover_minus_one_is_no_cover() { + let mut d = SearchDoc { + key: "/works/OL66554W".into(), + cover_i: Some(-1), + ..Default::default() + }; + assert_eq!(d.cover_url(), None); + d.cover_i = Some(14348537); + assert_eq!( + d.cover_url().as_deref(), + Some("https://covers.openlibrary.org/b/id/14348537-L.jpg") + ); + } + + #[test] + fn a_binding_needs_exactly_one_work_that_lists_the_etext() { + let one = serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671"] } + ]}); + assert_eq!(single_gutenberg_work(&one, 1342).as_deref(), Some("OL66554W")); + + let none = serde_json::json!({ "numFound": 0, "docs": [] }); + assert_eq!(single_gutenberg_work(&none, 84), None); + + // `limit=2` returned two docs: an unmerged duplicate, not a binding. + let two = serde_json::json!({ "numFound": 2, "docs": [ + { "key": "/works/OL1W", "id_project_gutenberg": ["11"] }, + { "key": "/works/OL2W", "id_project_gutenberg": ["11"] } + ]}); + assert_eq!(single_gutenberg_work(&two, 11), None); + + // More found than returned also counts as ambiguous. + let more = serde_json::json!({ "numFound": 3, "docs": [ + { "key": "/works/OL1W", "id_project_gutenberg": ["11"] } + ]}); + assert_eq!(single_gutenberg_work(&more, 11), None); + + // A doc that does not list the etext itself proves nothing. + let unlisted = serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL1W", "id_project_gutenberg": ["999"] } + ]}); + assert_eq!(single_gutenberg_work(&unlisted, 11), None); + + // An edition key is never bound as a work. + let edition = serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL12710531M", "id_project_gutenberg": ["11"] } + ]}); + assert_eq!(single_gutenberg_work(&edition, 11), None); + } + + #[test] + fn contact_is_made_header_safe() { + assert_eq!( + sanitize_contact(" ops@example.org\r\n(x) ").as_deref(), + Some("ops@example.orgx") + ); + assert_eq!(sanitize_contact("\n"), None); + assert!(user_agent(Some("ops@example.org")).ends_with("; ops@example.org)")); + // Must build without panicking. + let _ = OpenLibraryClient::new(budget(false), None, None, Some("a@b.c")); + } + + #[test] + fn a_dashboard_saved_contact_is_found_from_either_plugin_dir() { + let root = tempfile::tempdir().unwrap(); + let gutenberg = root.path().join("gutenberg"); + let openlibrary = root.path().join("openlibrary"); + std::fs::create_dir_all(&gutenberg).unwrap(); + std::fs::create_dir_all(&openlibrary).unwrap(); + // The SDK's config page saves under the lowest upstream id. + std::fs::write( + gutenberg.join("config.json"), + r#"{"contact":"ops@example.org"}"#, + ) + .unwrap(); + assert_eq!(load_contact(&openlibrary).as_deref(), Some("ops@example.org")); + assert_eq!(load_contact(&gutenberg).as_deref(), Some("ops@example.org")); + } + + /// A cached miss is not re-asked inside its window, and a failure is never + /// cached at all. + #[tokio::test] + async fn binding_lookups_cache_answers_but_never_failures() { + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "id_project_gutenberg:84")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 0, "docs": [] })), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "id_project_gutenberg:11")) + .respond_with(ResponseTemplate::new(503)) + .expect(2) + .mount(&server) + .await; + + let dir = tempfile::tempdir().unwrap(); + let cache = MidhashCache::open(dir.path()).unwrap(); + let c = OpenLibraryClient::new(RateBudget::new(100.0, 100.0), Some(cache), Some(server.uri()), None); + + assert_eq!(c.gutenberg_work(84, SEARCH_DEADLINE).await.unwrap(), None); + assert_eq!(c.gutenberg_work(84, SEARCH_DEADLINE).await.unwrap(), None); + assert!(c.gutenberg_work(11, SEARCH_DEADLINE).await.is_err()); + assert!(c.gutenberg_work(11, SEARCH_DEADLINE).await.is_err()); + } +} diff --git a/feeder-plugin/book-feeder/tests/contract.rs b/feeder-plugin/book-feeder/tests/contract.rs index df608a8..f306a98 100644 --- a/feeder-plugin/book-feeder/tests/contract.rs +++ b/feeder-plugin/book-feeder/tests/contract.rs @@ -1,24 +1,36 @@ //! End-to-end contract test for the feeder HTTP surface: boots the real -//! `serve_feeders` router (book-feeder hosting GutenbergPlugin) on an ephemeral -//! port, pointed at a wiremock Gutendex, and drives `/manifest`, `/query`, and -//! `/compute` over HTTP. Deterministic — no live Gutendex. +//! `serve_feeders` router (book-feeder hosting GutenbergPlugin AND +//! OpenLibraryPlugin, as `main.rs` does) on an ephemeral port, pointed at one +//! wiremock server that plays Gutendex, the Gutenberg RDF host and Open Library, +//! and drives `/manifest`, `/query`, and `/compute` over HTTP. Deterministic — +//! no live upstream. use book_feeder::gutenberg::GutenbergPlugin; +use book_feeder::openlibrary::OpenLibraryPlugin; +use book_feeder::openlibrary_client; use meta_feeder_sdk::{ - configure_plugins, hash, router, ComputeRequest, ComputeResponse, GatewayQuery, HashKindDto, - ManifestResponse, QueryRequest, QueryResponse, + configure_plugins, hash, router, ComputeRequest, ComputeResponse, FeederPlugin, GatewayQuery, + HashKindDto, ManifestResponse, QueryRequest, QueryResponse, }; -use wiremock::matchers::{method, path}; +use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, ResponseTemplate}; -/// Spawn the feeder router (Gutenberg pointed at `upstream`) on an ephemeral -/// port; return its base URL. The TempDir is returned so the per-plugin cache -/// outlives the test. +/// Spawn the feeder router (every upstream pointed at `upstream`) on an +/// ephemeral port; return its base URL. The TempDir is returned so the +/// per-plugin cache outlives the test. async fn spawn_feeder(upstream: &MockServer) -> (String, tempfile::TempDir) { let dir = tempfile::tempdir().expect("tempdir"); - let plugin = GutenbergPlugin::with_base_url(upstream.uri()); - let plugins = - configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure plugins"); + let budget = openlibrary_client::budget(false); + let plugins: Vec> = vec![ + Box::new( + GutenbergPlugin::with_base_url(upstream.uri()) + .with_rdf_base_url(upstream.uri()) + .with_openlibrary_base_url(upstream.uri()) + .with_openlibrary_budget(budget.clone()), + ), + Box::new(OpenLibraryPlugin::with_api_base(upstream.uri()).with_budget(budget)), + ]; + let plugins = configure_plugins(plugins, dir.path()).expect("configure plugins"); let app = router(plugins, "book-feeder-test".to_string(), dir.path()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") @@ -44,7 +56,7 @@ fn books_json() -> serde_json::Value { } #[tokio::test] -async fn manifest_lists_book_capabilities() { +async fn manifest_lists_both_upstreams() { let upstream = MockServer::start().await; let (base, _dir) = spawn_feeder(&upstream).await; @@ -57,21 +69,36 @@ async fn manifest_lists_book_capabilities() { .await .expect("decode manifest"); - assert_eq!(manifest.plugins.len(), 1); - let p = &manifest.plugins[0]; - assert_eq!(p.id, "gutenberg"); - assert_eq!(p.served_file_types, vec!["document".to_string()]); - assert_eq!(p.served_content_kinds, vec!["book".to_string()]); + assert_eq!(manifest.plugins.len(), 2); + let gutenberg = manifest.plugins.iter().find(|p| p.id == "gutenberg").expect("gutenberg"); + assert_eq!(gutenberg.served_file_types, vec!["document".to_string()]); + assert_eq!(gutenberg.served_content_kinds, vec!["book".to_string()]); + let openlibrary = manifest + .plugins + .iter() + .find(|p| p.id == "openlibrary") + .expect("openlibrary"); + assert_eq!(openlibrary.served_file_types, vec!["card".to_string()]); + assert_eq!(openlibrary.served_content_kinds, vec!["book".to_string()]); } #[tokio::test] -async fn query_returns_gutenberg_records() { +async fn query_returns_gutenberg_records_with_their_open_library_claim() { let upstream = MockServer::start().await; Mock::given(method("GET")) .and(path("/books/")) .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) .mount(&upstream) .await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "id_project_gutenberg:1342")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "numFound": 1, + "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671"] }] + }))) + .mount(&upstream) + .await; let (base, _dir) = spawn_feeder(&upstream).await; let resp: QueryResponse = reqwest::Client::new() @@ -90,9 +117,75 @@ async fn query_returns_gutenberg_records() { assert_eq!(resp.records.len(), 1); let r = &resp.records[0]; + let f = |k: &str| r.fields.get(k).map(String::as_str); assert_eq!(r.upstream_id, "gutenberg"); - assert_eq!(r.fields.get("gutenbergid").map(String::as_str), Some("1342")); - assert_eq!(r.fields.get("contentKind").map(String::as_str), Some("book")); + assert_eq!(f("gutenbergid"), Some("1342")); + assert_eq!(f("contentKind"), Some("book")); + assert_eq!(f("authors/Jane Austen"), Some("true")); + assert_eq!(f("languages/eng"), Some("true")); + // The transient claim the gateway transcribes into + // `anchoredBy/openlibrary:OL66554W`. + assert_eq!(f("openlibraryid"), Some("OL66554W")); + assert_eq!(f("anchored"), Some("true")); + assert_eq!(f("anchorSource"), Some("openlibrary")); + assert_eq!(f("anchorMethod"), None, "an identifier link is not a title match"); +} + +#[tokio::test] +async fn query_returns_an_open_library_card() { + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "pride")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "numFound": 1, + "docs": [{ + "key": "/works/OL66554W", + "title": "Pride and Prejudice", + "author_name": ["Jane Austen"], + "cover_i": 14348537, + "first_publish_year": 1813 + }] + }))) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/works/OL66554W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "key": "/works/OL66554W", + "description": "A novel of manners." + }))) + .mount(&upstream) + .await; + let (base, _dir) = spawn_feeder(&upstream).await; + + let mut query = GatewayQuery::from_free_text("pride"); + query.filters.insert("fileType".into(), vec!["card".into()]); + let resp: QueryResponse = reqwest::Client::new() + .post(format!("{base}/query")) + .json(&QueryRequest { + upstream_id: "openlibrary".to_string(), + query, + max_results: 10, + }) + .send() + .await + .expect("POST /query") + .json() + .await + .expect("decode query response"); + + assert_eq!(resp.records.len(), 1); + let r = &resp.records[0]; + assert_eq!(r.upstream_id, "openlibrary"); + assert_eq!(r.record_id, "OL66554W"); + assert_eq!(r.fields.get("fileType").map(String::as_str), Some("card")); + assert_eq!( + r.fields + .get("cids/bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx") + .map(String::as_str), + Some("true") + ); } #[tokio::test] @@ -151,6 +244,57 @@ async fn compute_returns_outcome_with_inline_epub_bytes() { assert_eq!(hash::compute_ipfs_cid(&decoded), expected_cid); } +#[tokio::test] +async fn compute_for_an_open_library_card_is_a_byteless_card_locator() { + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "key:/works/OL45804W")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "numFound": 1, + "docs": [{ + "key": "/works/OL45804W", + "title": "Fantastic Mr Fox", + "author_name": ["Roald Dahl"], + "cover_i": 6498519, + "first_publish_year": 1970 + }] + }))) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/works/OL45804W.json")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "key": "/works/OL45804W", + "description": "A clever fox outwits three farmers." + }))) + .mount(&upstream) + .await; + let (base, _dir) = spawn_feeder(&upstream).await; + + let resp: ComputeResponse = reqwest::Client::new() + .post(format!("{base}/compute")) + .json(&ComputeRequest { + upstream_id: "openlibrary".to_string(), + record_id: "OL45804W".to_string(), + }) + .send() + .await + .expect("POST /compute") + .json() + .await + .expect("decode compute response"); + + assert_eq!(resp.outcomes.len(), 1); + let o = &resp.outcomes[0]; + assert_eq!(o.hash, "bagdsaaaubnxxazlonruwe4tboj4u6tbugu4dancx"); + assert_eq!(o.hash_kind, HashKindDto::CardLocator); + assert!(o.bytes_b64.is_none()); + assert!(o.file_extension.is_none()); + let record = o.record.as_ref().expect("the card record rides the outcome"); + assert_eq!(record.fields.get("title").map(String::as_str), Some("Fantastic Mr Fox")); +} + #[tokio::test] async fn query_unknown_upstream_is_404() { let upstream = MockServer::start().await;