diff --git a/feeder-plugin/book-feeder/src/gutenberg.rs b/feeder-plugin/book-feeder/src/gutenberg.rs index 2201b94..83743cd 100644 --- a/feeder-plugin/book-feeder/src/gutenberg.rs +++ b/feeder-plugin/book-feeder/src/gutenberg.rs @@ -32,7 +32,17 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth}; use serde::Deserialize; use tracing::{debug, warn}; -use crate::openlibrary_client::{self, OpenLibraryClient}; +use crate::openlibrary::first_filter; +use crate::openlibrary_client::{self, Etext, OpenLibraryClient}; + +/// An edition's Open Library work, and how it was found. +#[derive(Debug, Clone)] +struct Binding { + work: String, + /// Bound by title and author ([`OpenLibraryClient::title_work`]) rather than + /// by an identifier Open Library records: graded Matched, not Anchored. + by_title: bool, +} /// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`] /// for tests + private mirrors. @@ -390,31 +400,72 @@ impl GutenbergPlugin { .await } + /// The Gutendex search a work page's query stands for. + /// + /// meta-read's book page asks for `openlibraryid:OL…W` with no free text. + /// That used to fall through to the catalogue browse — the 32 most + /// downloaded books — and the gateway kept those bound to the work, so a + /// book outside that list never showed an edition on its own page. Searching + /// for the work's main title and its first author's surname finds it + /// wherever it ranks; the gateway still keeps only the records bound to the + /// work. `None` (no work named, Open Library off or failing) keeps the browse. + async fn work_search_text(&self, query: &GatewayQuery) -> Option { + let work = first_filter(query, "openlibraryid")?.to_ascii_uppercase(); + if !openlibrary_client::is_work_id(&work) { + return None; + } + let client = self.openlibrary.as_ref()?; + let lookup = client.doc_by_work_id(&work, openlibrary_client::BINDING_DEADLINE); + let doc = tokio::time::timeout(BINDING_TIMEOUT, lookup).await.ok()?.ok()??; + let title = openlibrary_client::main_title(doc.title.as_deref()?); + if title.is_empty() { + return None; + } + let author = doc.author_name.first().map(|a| openlibrary_client::surname(a)).unwrap_or_default(); + Some(format!("{title} {author}").trim().to_string()) + } + /// Each result's Open Library work, through the Project Gutenberg id Open - /// Library records (reading-model §2.4). Aligned with the input slice; - /// `None` means no binding — none recorded, ambiguous, not configured, over - /// budget, or the lookup failed. + /// Library records (reading-model §2.4), or, for an etext Open Library lists + /// under no work, by title and author (`by_title`, graded Matched). Aligned + /// with the input slice; `None` means no binding — ambiguous, no title match, + /// not configured, over budget, or the lookup failed. /// /// ⚠ **A failure never blocks or drops a record.** The binding upgrades an /// edition that is already readable. Every failure is left uncached, so the /// next query that surfaces the book tries again; the budget deadline is /// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the /// first handful of books and the cache fills the rest over later queries. - async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec> { + async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec> { use futures::stream::{self, StreamExt}; const BINDING_CONCURRENCY: usize = 4; let Some(client) = self.openlibrary.as_ref() else { return vec![None; books.len()]; }; - // Owned ids, not `books.iter().map(..)`: a borrowing closure inside the + // Owned values, not `books.iter().map(..)`: a borrowing closure inside the // stream makes `handle_query`'s future fail the `async_trait` lifetime // check ("FnOnce is not general enough"). - let ids: Vec = books.iter().map(|b| b.id).collect(); - stream::iter(ids) - .map(|id| async move { - let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE); + let wanted: Vec<(u64, String, String)> = books + .iter() + .map(|b| (b.id, b.title.clone(), b.authors.first().map(|a| a.name.clone()).unwrap_or_default())) + .collect(); + stream::iter(wanted) + .map(|(id, title, author)| async move { + // The identifier first; a title match only for an etext Open + // Library lists under no work at all. + let lookup = async { + let deadline = openlibrary_client::BINDING_DEADLINE; + Ok::<_, GatewayError>(match client.gutenberg_etext(id, deadline).await? { + Etext::One(work) => Some(Binding { work, by_title: false }), + Etext::Several => None, + Etext::Unlisted => client + .title_work(&title, &author, deadline) + .await? + .map(|work| Binding { work, by_title: true }), + }) + }; match tokio::time::timeout(BINDING_TIMEOUT, lookup).await { - Ok(Ok(work)) => work, + Ok(Ok(binding)) => binding, Ok(Err(e)) => { debug!( target: "meta-share::gateway", @@ -507,7 +558,16 @@ impl FeederPlugin for GutenbergPlugin { // everything, already ordered by download count (identical to // `?sort=popular`), so the row leads with Pride and Prejudice rather than // with whatever happens to contain punctuation. - let q = query.free_text.trim(); + // A work page's query names its Open Library work and no text: search + // for that work's title and author instead of browsing. See + // `work_search_text`. + let free = query.free_text.trim(); + let work_text = if free.is_empty() { + self.work_search_text(query).await + } else { + None + }; + let q = work_text.as_deref().unwrap_or(free); let params: Vec<(&str, &str)> = if q.is_empty() { Vec::new() } else { @@ -650,7 +710,7 @@ fn into_discovery_record( book: GutendexBook, cover_cid: Option, bibrec: BTreeMap, - openlibrary_work: Option, + openlibrary_work: Option, ) -> DiscoveryRecord { let GutendexBook { id, @@ -803,16 +863,25 @@ fn into_discovery_record( let _ = download_count; fields.insert("fileName".into(), format!("gutenberg-{id}.epub")); - // The work binding and its trust claim (reading-model §2.4). Written only on - // an identifier link Open Library itself records, so the claim is the plain - // Anchored form: `anchored` + `anchorSource`, and deliberately NO - // `anchorMethod` — that marks a title match, which this never is. The + // The work binding and its trust claim (reading-model §2.4). On an + // identifier link Open Library itself records, the claim is the plain + // Anchored form: `anchored` + `anchorSource`, and NO `anchorMethod` — that + // marks a title match. The // gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and // strips the transient keys before storing. - if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) { - fields.insert("openlibraryid".into(), work); + // + // A title binding (an etext Open Library lists under no work) carries the + // same claim plus `anchorMethod=title`: the gateway then records + // `matchedBy/openlibrary:OL…W` instead of `anchoredBy/`, and meta-read grades + // the edition Matched — listed on its work's page, ranked after any + // identifier-bound edition. + if let Some(binding) = openlibrary_work.filter(|b| openlibrary_client::is_work_id(&b.work)) { + fields.insert("openlibraryid".into(), binding.work); fields.insert("anchored".into(), "true".into()); fields.insert("anchorSource".into(), "openlibrary".into()); + if binding.by_title { + fields.insert("anchorMethod".into(), "title".into()); + } } for (k, v) in bibrec { @@ -1678,6 +1747,98 @@ mod tests { } } + /// Open Library's answer to `title=…&author=…` for Frankenstein: several + /// works by Mary Shelley share the title; the most published is the novel. + fn frankenstein_by_title() -> Mock { + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("title", "frankenstein")) + .and(query_param("author", "shelley")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 3, "docs": [ + { "key": "/works/OL26860869W", "title": "Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 84 }, + { "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"], "edition_count": 2187 }, + { "key": "/works/OL25595002W", "title": "Mary Shelley's Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 90 } + ]}))) + } + + /// ⚠ Open Library lists etext 84 under no work, so the identifier rule alone + /// left the most read novel on Gutenberg unbound and its book page empty. + #[tokio::test] + async fn an_etext_no_work_lists_is_bound_by_title_and_graded_matched() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + binding( + 1342, + serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] } + ]}), + ) + .mount(&server) + .await; + binding(84, serde_json::json!({ "numFound": 0, "docs": [] })) + .mount(&server) + .await; + frankenstein_by_title().expect(1).mount(&server).await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("handle_query"); + + let pride = &records[0].fields; + assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W")); + assert!(!pride.contains_key("anchorMethod"), "an identifier binding stays Anchored"); + + let frankenstein = &records[1].fields; + assert_eq!(frankenstein.get("openlibraryid").map(String::as_str), Some("OL450063W")); + assert_eq!(frankenstein.get("anchored").map(String::as_str), Some("true")); + assert_eq!(frankenstein.get("anchorSource").map(String::as_str), Some("openlibrary")); + assert_eq!(frankenstein.get("anchorMethod").map(String::as_str), Some("title")); + } + + /// ⚠ A work page asks for its work and no text. That used to browse the 32 + /// most downloaded books, so a book outside them never reached its own page. + #[tokio::test] + async fn a_work_page_query_searches_for_the_works_title_and_author() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search.json")) + .and(query_param("q", "key:/works/OL450063W")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 1, "docs": [ + { "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"] } + ]}))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/books/")) + .and(query_param("search", "frankenstein shelley")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .expect(1) + .mount(&server) + .await; + binding(1342, serde_json::json!({ "numFound": 0, "docs": [] })) + .mount(&server) + .await; + binding(84, serde_json::json!({ "numFound": 0, "docs": [] })) + .mount(&server) + .await; + frankenstein_by_title().mount(&server).await; + + let (plugin, _dir) = configured_plugin_against(&server); + let mut query = GatewayQuery::from_free_text(""); + query.filters.insert("openlibraryid".into(), vec!["OL450063W".into()]); + query.filters.insert("contentKind".into(), vec!["book".into()]); + let records = plugin.handle_query(&query, 50).await.expect("handle_query"); + let frankenstein = records.iter().find(|r| r.record_id == "84").expect("etext 84 is found"); + assert_eq!(frankenstein.fields.get("openlibraryid").map(String::as_str), Some("OL450063W")); + assert_eq!(frankenstein.fields.get("anchorMethod").map(String::as_str), Some("title")); + } + /// ⚠ A binding failure never costs the record. #[tokio::test] async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() { diff --git a/feeder-plugin/book-feeder/src/openlibrary.rs b/feeder-plugin/book-feeder/src/openlibrary.rs index a8486a2..e688072 100644 --- a/feeder-plugin/book-feeder/src/openlibrary.rs +++ b/feeder-plugin/book-feeder/src/openlibrary.rs @@ -71,7 +71,7 @@ fn env_usize(key: &str, default: usize) -> usize { } /// First non-empty value for `key`. -fn first_filter(q: &GatewayQuery, key: &str) -> Option { +pub(crate) fn first_filter(q: &GatewayQuery, key: &str) -> Option { q.filters .get(key) .and_then(|v| v.first()) diff --git a/feeder-plugin/book-feeder/src/openlibrary_client.rs b/feeder-plugin/book-feeder/src/openlibrary_client.rs index fcd7239..74c10a2 100644 --- a/feeder-plugin/book-feeder/src/openlibrary_client.rs +++ b/feeder-plugin/book-feeder/src/openlibrary_client.rs @@ -614,6 +614,248 @@ impl OpenLibraryClient { self.cache_put(&key, &serde_json::json!({ "work": work })); Ok(work) } + + /// What Open Library records about an etext: the one work that lists it, + /// several works, or none. + /// + /// Same lookup as [`Self::gutenberg_work`], but it keeps the difference + /// between "several" and "none", which that one folds into `None`: only an + /// etext no work lists may be bound by title ([`Self::title_work`]). Several + /// works listing one etext is an unmerged duplicate, and guessing between + /// them is exactly what the identifier rule refuses. + pub async fn gutenberg_etext( + &self, + gutenberg_id: u64, + deadline: Duration, + ) -> Result { + let key = format!("openlibrary:etext:{gutenberg_id}"); + if let Some(c) = self.cache_get(&key) { + if let Some(etext) = Etext::from_cached(&c.body) { + let ttl = if matches!(etext, Etext::One(_)) { + BINDING_HIT_TTL_SECS + } else { + BINDING_MISS_TTL_SECS + }; + if now_secs().saturating_sub(c.at) <= ttl { + return Ok(etext); + } + } + } + let body = self + .get( + "/search.json", + &[ + ("q", format!("id_project_gutenberg:{gutenberg_id}")), + ("fields", "key,id_project_gutenberg".into()), + ("limit", "2".into()), + ], + deadline, + ) + .await?; + let etext = etext_of(&body, gutenberg_id); + self.cache_put(&key, &etext.to_cached()); + Ok(etext) + } + + /// The work a book is bound to by **title and author**, for an etext Open + /// Library lists under no work. + /// + /// Open Library's `id_project_gutenberg` links are patchy even for the most + /// read books — none of the editions of *Frankenstein; or, The Modern + /// Prometheus* (OL450063W) lists etext 84 — so the identifier rule alone left + /// them unbound, and their book pages, which ask for `openlibraryid:OL…W`, + /// empty. This is the weaker proof the trust model already has a grade for: + /// the caller stamps `anchorMethod=title`, the gateway records + /// `matchedBy/openlibrary:OL…W`, and meta-read ranks such an edition after an + /// identifier-bound one. + /// + /// Same main title ([`main_title`]), an author with the same surname, and, + /// among those, the work with the most editions — Open Library holds many + /// small works titled *Frankenstein* by Mary Shelley (adaptations, annotated + /// and school editions), and the canonical one is the one most published. + /// `None` when nothing matches or two works tie. + pub async fn title_work( + &self, + title: &str, + author: &str, + deadline: Duration, + ) -> Result, GatewayError> { + let (want, surname) = (main_title(title), surname(author)); + if want.is_empty() || surname.is_empty() { + return Ok(None); + } + let key = format!("openlibrary:titlework:{want}:{surname}"); + if let Some(c) = self.cache_get(&key) { + let work = c.body.get("work").and_then(Value::as_str).map(str::to_string); + let ttl = if work.is_some() { + BINDING_HIT_TTL_SECS + } else { + BINDING_MISS_TTL_SECS + }; + if now_secs().saturating_sub(c.at) <= ttl { + return Ok(work); + } + } + let body = self + .get( + "/search.json", + &[ + ("title", want.clone()), + ("author", surname.clone()), + ("fields", "key,title,author_name,edition_count".into()), + ("limit", "20".into()), + ], + deadline, + ) + .await?; + let work = best_title_work(&body, &want, &surname); + self.cache_put(&key, &serde_json::json!({ "work": work })); + Ok(work) + } +} + +/// What Open Library records about one Gutenberg etext. See +/// [`OpenLibraryClient::gutenberg_etext`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Etext { + /// Exactly one work lists it: an identifier binding, graded Anchored. + One(String), + /// Several works list it: an unmerged duplicate, so no binding at all. + Several, + /// No work lists it: the one case a title binding may fill. + Unlisted, +} + +impl Etext { + fn to_cached(&self) -> Value { + match self { + Etext::One(work) => serde_json::json!({ "etext": "one", "work": work }), + Etext::Several => serde_json::json!({ "etext": "several" }), + Etext::Unlisted => serde_json::json!({ "etext": "unlisted" }), + } + } + + fn from_cached(v: &Value) -> Option { + match v.get("etext")?.as_str()? { + "one" => Some(Etext::One(v.get("work")?.as_str()?.to_string())), + "several" => Some(Etext::Several), + "unlisted" => Some(Etext::Unlisted), + _ => None, + } + } +} + +fn etext_of(body: &Value, gutenberg_id: u64) -> Etext { + if let Some(work) = single_gutenberg_work(body, gutenberg_id) { + return Etext::One(work); + } + let docs = parse_docs(body.get("docs")); + let total = body + .get("numFound") + .or_else(|| body.get("num_found")) + .and_then(Value::as_u64) + .unwrap_or(docs.len() as u64); + if total == 0 && docs.is_empty() { + Etext::Unlisted + } else { + Etext::Several + } +} + +/// The part of a title two catalogues agree on: before any subtitle (`;`, `:`, +/// a bracket, or `, or`), lower-cased, letters and digits only. Gutenberg's +/// *Frankenstein; Or, The Modern Prometheus* and Open Library's *Frankenstein; +/// or, The Modern Prometheus* are both `frankenstein`. +pub fn main_title(raw: &str) -> String { + let lower = raw.to_lowercase(); + let head = lower + .split(|c: char| matches!(c, ';' | ':' | '(' | '[')) + .next() + .unwrap_or_default(); + let head = head.find(", or").map_or(head, |i| &head[..i]); + head.split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .collect::>() + .join(" ") +} + +/// The surname a name ends on, lower-cased, letters and digits only: Gutendex +/// writes `Shelley, Mary Wollstonecraft`, Open Library `Mary Shelley`; both give +/// `shelley`. +pub fn surname(raw: &str) -> String { + let raw = raw.trim(); + let last = match raw.split_once(',') { + Some((last, _)) => last, + None => raw.rsplit(char::is_whitespace).next().unwrap_or_default(), + }; + last.chars() + .filter(|c| c.is_alphanumeric()) + .collect::() + .to_lowercase() +} + +/// Does `name` carry `surname`? Compared with the separators removed, so a +/// compound surname (`Saint-Exupéry`) matches however it is spaced. +fn author_has_surname(name: &str, surname: &str) -> bool { + let flat = name + .chars() + .filter(|c| c.is_alphanumeric()) + .collect::() + .to_lowercase(); + !surname.is_empty() && flat.contains(surname) +} + +/// One document of the title search, with the edition count the choice rests on. +#[derive(Debug, Deserialize)] +struct TitleDoc { + #[serde(default)] + key: String, + #[serde(default)] + title: Option, + #[serde(default)] + author_name: Vec, + #[serde(default)] + edition_count: Option, +} + +/// Decide a title binding from a `title=…&author=…` response. See +/// [`OpenLibraryClient::title_work`]. +fn best_title_work(body: &Value, want: &str, surname: &str) -> Option { + let docs: Vec = body + .get("docs") + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|d| serde_json::from_value::(d.clone()).ok()) + .collect() + }) + .unwrap_or_default(); + let mut best: Option<(u64, String)> = None; + let mut tie = false; + for d in docs { + let Some(id) = d.key.strip_prefix("/works/").filter(|id| is_work_id(id)) else { + continue; + }; + if main_title(d.title.as_deref().unwrap_or_default()) != want { + continue; + } + if !d.author_name.iter().any(|a| author_has_surname(a, surname)) { + continue; + } + let n = d.edition_count.unwrap_or(0); + let top = best.as_ref().map(|(m, _)| *m); + if top.is_none_or(|m| n > m) { + best = Some((n, id.to_string())); + tie = false; + } else if top == Some(n) { + tie = true; + } + } + if tie { + None + } else { + best.map(|(_, id)| id) + } } /// Decide a binding from a `q=id_project_gutenberg:` response. @@ -670,6 +912,70 @@ fn clean_description(raw: &str) -> Option { mod tests { use super::*; + #[test] + fn a_main_title_drops_the_subtitle_case_and_punctuation() { + assert_eq!(main_title("Frankenstein; Or, The Modern Prometheus"), "frankenstein"); + assert_eq!(main_title("Frankenstein; or, The Modern Prometheus"), "frankenstein"); + assert_eq!(main_title("Frankenstein, or the Modern Prometheus"), "frankenstein"); + assert_eq!(main_title("Frankenstein (Oxford Playscripts)"), "frankenstein"); + assert_eq!(main_title("Alice's Adventures in Wonderland"), "alice s adventures in wonderland"); + assert_eq!(main_title(""), ""); + } + + #[test] + fn a_surname_reads_both_name_orders() { + assert_eq!(surname("Shelley, Mary Wollstonecraft"), "shelley"); + assert_eq!(surname("Mary Shelley"), "shelley"); + assert_eq!(surname(" Austen, Jane "), "austen"); + assert_eq!(surname("Saint-Exupéry, Antoine de"), "saintexupéry"); + assert!(author_has_surname("Antoine de Saint-Exupéry", "saintexupéry")); + assert!(!author_has_surname("Jason Cobley", "shelley")); + } + + fn title_doc(key: &str, title: &str, author: &str, editions: u64) -> Value { + serde_json::json!({ "key": key, "title": title, "author_name": [author], "edition_count": editions }) + } + + #[test] + fn the_title_binding_takes_the_most_published_work_of_that_title_and_author() { + let body = serde_json::json!({ "docs": [ + title_doc("/works/OL26860869W", "Frankenstein", "Mary Shelley", 84), + title_doc("/works/OL450063W", "Frankenstein; or, The Modern Prometheus", "Mary Shelley", 2187), + title_doc("/works/OL25595002W", "Mary Shelley's Frankenstein", "Mary Shelley", 9000), + title_doc("/works/OL15443537W", "Frankenstein", "Dean Koontz", 5000), + ]}); + // The 9000-edition work has another title, the 5000 one another author. + assert_eq!(best_title_work(&body, "frankenstein", "shelley").as_deref(), Some("OL450063W")); + } + + #[test] + fn a_tie_or_no_match_binds_nothing() { + let tie = serde_json::json!({ "docs": [ + title_doc("/works/OL1W", "Frankenstein", "Mary Shelley", 10), + title_doc("/works/OL2W", "Frankenstein", "Mary Shelley", 10), + ]}); + assert_eq!(best_title_work(&tie, "frankenstein", "shelley"), None); + let other = serde_json::json!({ "docs": [title_doc("/works/OL3W", "Dracula", "Bram Stoker", 99)] }); + assert_eq!(best_title_work(&other, "frankenstein", "shelley"), None); + let not_a_work = serde_json::json!({ "docs": [title_doc("/books/OL4M", "Frankenstein", "Mary Shelley", 99)] }); + assert_eq!(best_title_work(¬_a_work, "frankenstein", "shelley"), None); + } + + #[test] + fn an_etext_is_one_several_or_unlisted() { + let one = serde_json::json!({ "numFound": 1, "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }] }); + assert_eq!(etext_of(&one, 1342), Etext::One("OL66554W".into())); + let several = serde_json::json!({ "numFound": 2, "docs": [ + { "key": "/works/OL450063W", "id_project_gutenberg": ["84"] }, + { "key": "/works/OL18388W", "id_project_gutenberg": ["84"] } + ]}); + assert_eq!(etext_of(&several, 84), Etext::Several); + assert_eq!(etext_of(&serde_json::json!({ "numFound": 0, "docs": [] }), 84), Etext::Unlisted); + for e in [Etext::One("OL1W".into()), Etext::Several, Etext::Unlisted] { + assert_eq!(Etext::from_cached(&e.to_cached()), Some(e)); + } + } + #[test] fn work_ids_are_works_only() { assert!(is_work_id("OL66554W"));