6 changed files with 186 additions and 547 deletions
-31
View File
@@ -1,31 +0,0 @@
# Runs the feeder's tests on every push. The repo had only the image workflow,
# so nothing checked a change before it shipped.
name: Test
on:
push:
branches: ["**"]
jobs:
rust:
name: cargo test
runs-on: ubuntu-latest
# The job ran out of memory linking the test binaries (ld killed with
# signal 9, run 705 twice): no debug info, and at most two links at once.
# What is tested does not change.
env:
CARGO_PROFILE_DEV_DEBUG: "0"
CARGO_BUILD_JOBS: "2"
steps:
- uses: actions/checkout@v4
# rust-toolchain.toml pins the channel (with clippy); rustup installs it on
# first use, so the action only has to provide rustup itself.
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- uses: Swatinem/rust-cache@v2
- name: cargo test
run: cargo test --locked
- name: cargo clippy
run: cargo clippy --all-targets -- -D warnings
Generated
+1 -1
View File
@@ -792,7 +792,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "meta-feeder-sdk"
version = "1.7.0"
source = "git+https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
dependencies = [
"anyhow",
"async-trait",
+1 -1
View File
@@ -24,7 +24,7 @@ path = "src/main.rs"
# vendored copy this used to point at had already drifted behind the shared SDK
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
# way; see meta-feeder-anilist's manifest.
meta-feeder-sdk = { git = "https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk", tag = "v1.7.0" }
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
tokio = { version = "1.40", features = ["full"] }
axum = { version = "0.7", features = ["json"] }
+23 -192
View File
@@ -32,37 +32,7 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
use serde::Deserialize;
use tracing::{debug, warn};
use crate::openlibrary::first_filter;
use crate::openlibrary_client::{self, Etext, OpenLibraryClient};
/// An edition's Open Library work, and how it was found.
#[derive(Debug, Clone)]
struct Binding {
work: String,
/// Bound by title and author ([`OpenLibraryClient::title_work`]) rather than
/// by an identifier Open Library records: graded Matched, not Anchored.
by_title: bool,
}
/// One etext's binding: the identifier Open Library records, else — only for an
/// etext it lists under no work — the title match. Shared by the search path and
/// the redeem path so the two cannot bind the same book differently.
async fn bind(
client: &OpenLibraryClient,
id: u64,
title: &str,
author: &str,
deadline: std::time::Duration,
) -> Result<Option<Binding>, GatewayError> {
Ok(match client.gutenberg_etext(id, deadline).await? {
Etext::One(work) => Some(Binding { work, by_title: false }),
Etext::Several => None,
Etext::Unlisted => client
.title_work(title, author, deadline)
.await?
.map(|work| Binding { work, by_title: true }),
})
}
use crate::openlibrary_client::{self, OpenLibraryClient};
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
/// for tests + private mirrors.
@@ -420,60 +390,31 @@ impl GutenbergPlugin {
.await
}
/// The Gutendex search a work page's query stands for.
///
/// meta-read's book page asks for `openlibraryid:OL…W` with no free text.
/// That used to fall through to the catalogue browse — the 32 most
/// downloaded books — and the gateway kept those bound to the work, so a
/// book outside that list never showed an edition on its own page. Searching
/// for the work's main title and its first author's surname finds it
/// wherever it ranks; the gateway still keeps only the records bound to the
/// work. `None` (no work named, Open Library off or failing) keeps the browse.
async fn work_search_text(&self, query: &GatewayQuery) -> Option<String> {
let work = first_filter(query, "openlibraryid")?.to_ascii_uppercase();
if !openlibrary_client::is_work_id(&work) {
return None;
}
let client = self.openlibrary.as_ref()?;
let lookup = client.doc_by_work_id(&work, openlibrary_client::BINDING_DEADLINE);
let doc = tokio::time::timeout(BINDING_TIMEOUT, lookup).await.ok()?.ok()??;
let title = openlibrary_client::main_title(doc.title.as_deref()?);
if title.is_empty() {
return None;
}
let author = doc.author_name.first().map(|a| openlibrary_client::surname(a)).unwrap_or_default();
Some(format!("{title} {author}").trim().to_string())
}
/// Each result's Open Library work, through the Project Gutenberg id Open
/// Library records (reading-model §2.4), or, for an etext Open Library lists
/// under no work, by title and author (`by_title`, graded Matched). Aligned
/// with the input slice; `None` means no binding — ambiguous, no title match,
/// not configured, over budget, or the lookup failed.
/// Library records (reading-model §2.4). Aligned with the input slice;
/// `None` means no binding — none recorded, ambiguous, not configured, over
/// budget, or the lookup failed.
///
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
/// edition that is already readable. Every failure is left uncached, so the
/// next query that surfaces the book tries again; the budget deadline is
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
/// first handful of books and the cache fills the rest over later queries.
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<Binding>> {
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<String>> {
use futures::stream::{self, StreamExt};
const BINDING_CONCURRENCY: usize = 4;
let Some(client) = self.openlibrary.as_ref() else {
return vec![None; books.len()];
};
// Owned values, not `books.iter().map(..)`: a borrowing closure inside the
// Owned ids, not `books.iter().map(..)`: a borrowing closure inside the
// stream makes `handle_query`'s future fail the `async_trait` lifetime
// check ("FnOnce is not general enough").
let wanted: Vec<(u64, String, String)> = books
.iter()
.map(|b| (b.id, b.title.clone(), b.authors.first().map(|a| a.name.clone()).unwrap_or_default()))
.collect();
stream::iter(wanted)
.map(|(id, title, author)| async move {
let lookup = bind(client, id, &title, &author, openlibrary_client::BINDING_DEADLINE);
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
stream::iter(ids)
.map(|id| async move {
let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE);
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
Ok(Ok(binding)) => binding,
Ok(Ok(work)) => work,
Ok(Err(e)) => {
debug!(
target: "meta-share::gateway",
@@ -566,16 +507,7 @@ impl FeederPlugin for GutenbergPlugin {
// everything, already ordered by download count (identical to
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
// with whatever happens to contain punctuation.
// A work page's query names its Open Library work and no text: search
// for that work's title and author instead of browsing. See
// `work_search_text`.
let free = query.free_text.trim();
let work_text = if free.is_empty() {
self.work_search_text(query).await
} else {
None
};
let q = work_text.as_deref().unwrap_or(free);
let q = query.free_text.trim();
let params: Vec<(&str, &str)> = if q.is_empty() {
Vec::new()
} else {
@@ -636,9 +568,9 @@ impl FeederPlugin for GutenbergPlugin {
// Not user-facing, so the binding lookup may queue for the full search
// deadline. A failure still leaves the record unbound, never fails the
// compute.
let author = book.authors.first().map(|a| a.name.clone()).unwrap_or_default();
let work = match self.openlibrary.as_ref() {
Some(ol) => bind(ol, book.id, &book.title, &author, openlibrary_client::SEARCH_DEADLINE)
Some(ol) => ol
.gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE)
.await
.unwrap_or_else(|e| {
debug!(
@@ -712,13 +644,13 @@ impl FeederPlugin for GutenbergPlugin {
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
///
/// `openlibrary_work` is the binding from [`bind`]: the one work Open Library
/// records for this etext, or, when it records none, the title match.
/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]:
/// `Some` only when Open Library records exactly one work for this etext.
fn into_discovery_record(
book: GutendexBook,
cover_cid: Option<String>,
bibrec: BTreeMap<String, String>,
openlibrary_work: Option<Binding>,
openlibrary_work: Option<String>,
) -> DiscoveryRecord {
let GutendexBook {
id,
@@ -871,25 +803,16 @@ fn into_discovery_record(
let _ = download_count;
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
// The work binding and its trust claim (reading-model §2.4). On an
// identifier link Open Library itself records, the claim is the plain
// Anchored form: `anchored` + `anchorSource`, and NO `anchorMethod` — that
// marks a title match. The
// The work binding and its trust claim (reading-model §2.4). Written only on
// an identifier link Open Library itself records, so the claim is the plain
// Anchored form: `anchored` + `anchorSource`, and deliberately NO
// `anchorMethod` — that marks a title match, which this never is. The
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
// strips the transient keys before storing.
//
// A title binding (an etext Open Library lists under no work) carries the
// same claim plus `anchorMethod=title`: the gateway then records
// `matchedBy/openlibrary:OL…W` instead of `anchoredBy/`, and meta-read grades
// the edition Matched — listed on its work's page, ranked after any
// identifier-bound edition.
if let Some(binding) = openlibrary_work.filter(|b| openlibrary_client::is_work_id(&b.work)) {
fields.insert("openlibraryid".into(), binding.work);
if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) {
fields.insert("openlibraryid".into(), work);
fields.insert("anchored".into(), "true".into());
fields.insert("anchorSource".into(), "openlibrary".into());
if binding.by_title {
fields.insert("anchorMethod".into(), "title".into());
}
}
for (k, v) in bibrec {
@@ -1755,98 +1678,6 @@ mod tests {
}
}
/// Open Library's answer to `title=…&author=…` for Frankenstein: several
/// works by Mary Shelley share the title; the most published is the novel.
fn frankenstein_by_title() -> Mock {
Mock::given(method("GET"))
.and(path("/search.json"))
.and(query_param("title", "frankenstein"))
.and(query_param("author", "shelley"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 3, "docs": [
{ "key": "/works/OL26860869W", "title": "Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 84 },
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"], "edition_count": 2187 },
{ "key": "/works/OL25595002W", "title": "Mary Shelley's Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 90 }
]})))
}
/// ⚠ Open Library lists etext 84 under no work, so the identifier rule alone
/// left the most read novel on Gutenberg unbound and its book page empty.
#[tokio::test]
async fn an_etext_no_work_lists_is_bound_by_title_and_graded_matched() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/books/"))
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
.mount(&server)
.await;
binding(
1342,
serde_json::json!({ "numFound": 1, "docs": [
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }
]}),
)
.mount(&server)
.await;
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
.mount(&server)
.await;
frankenstein_by_title().expect(1).mount(&server).await;
let (plugin, _dir) = configured_plugin_against(&server);
let records = plugin
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
.await
.expect("handle_query");
let pride = &records[0].fields;
assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W"));
assert!(!pride.contains_key("anchorMethod"), "an identifier binding stays Anchored");
let frankenstein = &records[1].fields;
assert_eq!(frankenstein.get("openlibraryid").map(String::as_str), Some("OL450063W"));
assert_eq!(frankenstein.get("anchored").map(String::as_str), Some("true"));
assert_eq!(frankenstein.get("anchorSource").map(String::as_str), Some("openlibrary"));
assert_eq!(frankenstein.get("anchorMethod").map(String::as_str), Some("title"));
}
/// ⚠ A work page asks for its work and no text. That used to browse the 32
/// most downloaded books, so a book outside them never reached its own page.
#[tokio::test]
async fn a_work_page_query_searches_for_the_works_title_and_author() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/search.json"))
.and(query_param("q", "key:/works/OL450063W"))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 1, "docs": [
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"] }
]})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/books/"))
.and(query_param("search", "frankenstein shelley"))
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
.expect(1)
.mount(&server)
.await;
binding(1342, serde_json::json!({ "numFound": 0, "docs": [] }))
.mount(&server)
.await;
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
.mount(&server)
.await;
frankenstein_by_title().mount(&server).await;
let (plugin, _dir) = configured_plugin_against(&server);
let mut query = GatewayQuery::from_free_text("");
query.filters.insert("openlibraryid".into(), vec!["OL450063W".into()]);
query.filters.insert("contentKind".into(), vec!["book".into()]);
let records = plugin.handle_query(&query, 50).await.expect("handle_query");
let frankenstein = records.iter().find(|r| r.record_id == "84").expect("etext 84 is found");
assert_eq!(frankenstein.fields.get("openlibraryid").map(String::as_str), Some("OL450063W"));
assert_eq!(frankenstein.fields.get("anchorMethod").map(String::as_str), Some("title"));
}
/// ⚠ A binding failure never costs the record.
#[tokio::test]
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
+131 -14
View File
@@ -71,7 +71,7 @@ fn env_usize(key: &str, default: usize) -> usize {
}
/// First non-empty value for `key`.
pub(crate) fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
q.filters
.get(key)
.and_then(|v| v.first())
@@ -169,14 +169,33 @@ fn foreign_identity_filter(q: &GatewayQuery) -> Option<&str> {
/// what passes, permanently. A card that lacks another filter's key does not
/// match it, which is the truth.
pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option<DiscoveryRecord> {
to_record_with(doc, Some(description), query, false)
}
/// [`to_record`] with the quality bar as a choice.
///
/// `lenient = false` is the bar every search and browse row keeps: a cover **and**
/// a description, or no card. `lenient = true` is for a lookup that names one work
/// by `openlibraryid` (a work page): the work is already known to be wanted, so a
/// title is enough. Its cover may come from the work's ISBNs when the work has none
/// of its own, and a missing description is simply left out.
pub fn to_record_with(
doc: &SearchDoc,
description: Option<&str>,
query: &GatewayQuery,
lenient: bool,
) -> Option<DiscoveryRecord> {
let id = doc.work_id()?;
let cid = card_cid(id)?;
let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?;
let poster_url = doc.cover_url()?;
let description = description.trim();
if description.is_empty() {
return None;
}
let description = description.map(str::trim).filter(|d| !d.is_empty());
let poster_url = if lenient {
doc.cover_url().or_else(|| doc.isbn_cover_url())
} else {
// Strict: both are required.
description?;
Some(doc.cover_url()?)
};
let mut fields: BTreeMap<String, String> = BTreeMap::new();
@@ -200,7 +219,9 @@ pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Op
// `description/{lang3}`, never a flat `description` — the flat key is indexed
// but never written (METADATA_KEYS.md §14.6).
fields.insert("description/eng".into(), description.to_string());
if let Some(d) = description {
fields.insert("description/eng".into(), d.to_string());
}
let mut seen = HashSet::new();
let authors: Vec<&str> = doc
@@ -224,7 +245,9 @@ pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Op
// ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS
// path fetches it, seeds the bytes and writes the content cid back.
fields.insert("poster_url".into(), poster_url);
if let Some(url) = poster_url {
fields.insert("poster_url".into(), url);
}
fields.insert(
"sourceUrl".into(),
@@ -257,13 +280,14 @@ async fn project(
query: &GatewayQuery,
n: usize,
deadline: Duration,
lenient: bool,
) -> Vec<DiscoveryRecord> {
let mut seen = HashSet::new();
let candidates: Vec<SearchDoc> = docs
.into_iter()
.filter(|d| {
d.work_id().is_some()
&& d.cover_url().is_some()
&& (lenient || d.cover_url().is_some())
&& d.title.as_deref().is_some_and(|t| !t.trim().is_empty())
})
.filter(|d| seen.insert(d.key.clone()))
@@ -286,7 +310,7 @@ async fn project(
debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped");
return None;
}
to_record(&doc, w.description.as_deref()?, query)
to_record_with(&doc, w.description.as_deref(), query, lenient)
}
Ok(None) => None,
Err(e) => {
@@ -437,7 +461,9 @@ impl FeederPlugin for OpenLibraryPlugin {
let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else {
return Ok(Vec::new());
};
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE).await);
// Lenient: an explicit work lookup keeps the card even without a
// description, and finds a cover by ISBN when the work has none.
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE, true).await);
}
// A curated row.
@@ -455,7 +481,7 @@ impl FeederPlugin for OpenLibraryPlugin {
.await?
}
};
return Ok(project(client, docs, query, n, BROWSE_DEADLINE).await);
return Ok(project(client, docs, query, n, BROWSE_DEADLINE, false).await);
}
// Free text.
@@ -470,7 +496,7 @@ impl FeederPlugin for OpenLibraryPlugin {
let docs = client
.search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE)
.await?;
Ok(project(client, docs, query, n, SEARCH_DEADLINE).await)
Ok(project(client, docs, query, n, SEARCH_DEADLINE, false).await)
}
/// A card has no bytes: the record IS the payload.
@@ -489,7 +515,8 @@ impl FeederPlugin for OpenLibraryPlugin {
// different work (a merge) is not this cid's record.
let record = match self.client.as_ref() {
Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await {
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE)
// Lenient, like the `openlibraryid` lookup that minted this card.
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE, true)
.await
.into_iter()
.find(|r| r.record_id == id),
@@ -545,6 +572,7 @@ mod tests {
cover_i: Some(14348537),
first_publish_year: Some(1813),
id_project_gutenberg: vec![],
isbn: vec![],
}
}
@@ -651,6 +679,95 @@ mod tests {
assert!(to_record(&edition, "A novel.", &q("", &[])).is_none());
}
#[test]
fn a_lenient_card_needs_only_a_title_and_finds_a_cover_by_isbn() {
let mut d = pride();
d.cover_i = None;
d.isbn = vec!["1955252092".into(), "9781955252096".into()];
// Strict still refuses it, whatever the description.
assert!(to_record_with(&d, Some("A novel."), &q("", &[]), false).is_none());
// Lenient mints it with the ISBN-13 cover and no description at all.
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
assert_eq!(
r.fields["poster_url"],
"https://covers.openlibrary.org/b/isbn/9781955252096-L.jpg?default=false"
);
assert!(!r.fields.keys().any(|k| k.starts_with("description")));
assert_eq!(r.fields["openlibraryid"], "OL66554W");
assert_eq!(r.fields["fileType"], "card");
// Same address as the strict card for the same work.
let strict = to_record(&pride(), "A novel.", &q("", &[])).unwrap();
let key = |r: &DiscoveryRecord| r.fields.keys().find(|k| k.starts_with("cids/")).cloned();
assert_eq!(key(&r), key(&strict));
}
#[test]
fn a_lenient_card_with_no_cover_anywhere_has_no_poster_rather_than_no_card() {
let mut d = pride();
d.cover_i = None;
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
assert!(!r.fields.contains_key("poster_url"));
}
#[test]
fn isbn_cover_prefers_an_isbn13_and_ignores_junk() {
let mut d = SearchDoc::default();
d.isbn = vec!["not-an-isbn".into(), "0306406152".into(), "9780306406157".into()];
assert_eq!(
d.isbn_cover_url().as_deref(),
Some("https://covers.openlibrary.org/b/isbn/9780306406157-L.jpg?default=false")
);
d.isbn = vec!["0306406152".into()];
assert!(d.isbn_cover_url().unwrap().contains("0306406152"));
d.isbn = vec!["x".into()];
assert_eq!(d.isbn_cover_url(), None);
}
#[tokio::test]
async fn a_named_work_without_cover_or_description_still_gets_its_card_but_a_search_does_not() {
let server = MockServer::start().await;
let doc = json!({
"key": "/works/OL39284292W",
"title": "Mother of Learning",
"author_name": ["nobody103"],
"first_publish_year": 2021,
"language": ["eng"],
"isbn": ["9781955252096"]
});
// The lookup by id and the free-text search both find the same index doc.
Mock::given(method("GET"))
.and(path("/search.json"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"numFound": 1, "docs": [doc]})))
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/works/OL39284292W.json"))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"key": "/works/OL39284292W", "title": "Mother of Learning"
})))
.mount(&server)
.await;
let (p, _dir) = plugin_against(&server).await;
let by_id = p
.handle_query(
&q("", &[("openlibraryid", "OL39284292W"), ("contentKind", "book")]),
10,
)
.await
.unwrap();
assert_eq!(by_id.len(), 1, "the work page lookup yields a card");
assert_eq!(by_id[0].fields["title"], "Mother of Learning");
assert!(by_id[0].fields["poster_url"].contains("/b/isbn/9781955252096-L.jpg"));
assert!(!by_id[0].fields.keys().any(|k| k.starts_with("description")));
let by_search = p
.handle_query(&q("mother of learning", &[("fileType", "card"), ("contentKind", "book")]), 10)
.await
.unwrap();
assert!(by_search.is_empty(), "search rows keep the strict bar");
}
#[test]
fn echoes_browse_markers_but_no_other_filter() {
let r = to_record(
@@ -30,6 +30,9 @@ pub const DEFAULT_API_BASE: &str = "https://openlibrary.org";
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
/// `poster_url` names, so one size is published, not three.
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
/// Cover by ISBN. Open Library often holds an *edition's* cover for a work whose
/// own `covers` list is empty.
const COVERS_ISBN_BASE: &str = "https://covers.openlibrary.org/b/isbn";
/// Config key (and env seed) for the operator contact sent in the User-Agent.
pub const CONTACT_KEY: &str = "contact";
@@ -58,6 +61,9 @@ const RETRY_BACKOFF_MS: u64 = 250;
/// The field selection for every doc-shaped request, written once so a card
/// resolved by search and the same card resolved by id are byte-identical.
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
/// A lookup of one named work also asks for its ISBNs, so a cover can be found by
/// ISBN. Not on the shared list: a search row would carry hundreds of ISBNs per doc.
const DOC_BY_ID_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language,isbn";
/// Cache lifetimes.
///
@@ -199,6 +205,10 @@ pub struct SearchDoc {
pub first_publish_year: Option<i32>,
#[serde(default)]
pub id_project_gutenberg: Vec<String>,
/// ISBNs of the work's editions. Only requested for an `openlibraryid`
/// lookup (see [`DOC_BY_ID_FIELDS`]); empty everywhere else.
#[serde(default)]
pub isbn: Vec<String>,
}
impl SearchDoc {
@@ -214,6 +224,22 @@ impl SearchDoc {
.filter(|c| *c > 0)
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
}
/// A cover looked up by one of the work's ISBNs (an ISBN-13 when there is one),
/// for a work with no cover of its own.
///
/// `?default=false` makes Open Library answer 404 when it has no image for the
/// ISBN, instead of a 1×1 placeholder that would be seeded as the poster.
pub fn isbn_cover_url(&self) -> Option<String> {
let digits_only = |s: &&String| s.chars().all(|c| c.is_ascii_digit());
let isbn = self
.isbn
.iter()
.filter(digits_only)
.find(|s| s.len() == 13)
.or_else(|| self.isbn.iter().filter(digits_only).find(|s| s.len() == 10))?;
Some(format!("{COVERS_ISBN_BASE}/{isbn}-L.jpg?default=false"))
}
}
/// What a work JSON contributes to a card.
@@ -440,12 +466,13 @@ impl OpenLibraryClient {
) -> Result<Option<SearchDoc>, GatewayError> {
let body = self
.get_cached(
&format!("openlibrary:key:{id}"),
// `key2`: entries cached before ISBNs were requested lack them.
&format!("openlibrary:key2:{id}"),
WORK_TTL_SECS,
"/search.json",
&[
("q", format!("key:/works/{id}")),
("fields", SEARCH_FIELDS.into()),
("fields", DOC_BY_ID_FIELDS.into()),
("limit", "1".into()),
],
deadline,
@@ -493,6 +520,7 @@ impl OpenLibraryClient {
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
first_publish_year: None,
id_project_gutenberg: Vec::new(),
isbn: Vec::new(),
}))
}
@@ -614,248 +642,6 @@ impl OpenLibraryClient {
self.cache_put(&key, &serde_json::json!({ "work": work }));
Ok(work)
}
/// What Open Library records about an etext: the one work that lists it,
/// several works, or none.
///
/// Same lookup as [`Self::gutenberg_work`], but it keeps the difference
/// between "several" and "none", which that one folds into `None`: only an
/// etext no work lists may be bound by title ([`Self::title_work`]). Several
/// works listing one etext is an unmerged duplicate, and guessing between
/// them is exactly what the identifier rule refuses.
pub async fn gutenberg_etext(
&self,
gutenberg_id: u64,
deadline: Duration,
) -> Result<Etext, GatewayError> {
let key = format!("openlibrary:etext:{gutenberg_id}");
if let Some(c) = self.cache_get(&key) {
if let Some(etext) = Etext::from_cached(&c.body) {
let ttl = if matches!(etext, Etext::One(_)) {
BINDING_HIT_TTL_SECS
} else {
BINDING_MISS_TTL_SECS
};
if now_secs().saturating_sub(c.at) <= ttl {
return Ok(etext);
}
}
}
let body = self
.get(
"/search.json",
&[
("q", format!("id_project_gutenberg:{gutenberg_id}")),
("fields", "key,id_project_gutenberg".into()),
("limit", "2".into()),
],
deadline,
)
.await?;
let etext = etext_of(&body, gutenberg_id);
self.cache_put(&key, &etext.to_cached());
Ok(etext)
}
/// The work a book is bound to by **title and author**, for an etext Open
/// Library lists under no work.
///
/// Open Library's `id_project_gutenberg` links are patchy even for the most
/// read books — none of the editions of *Frankenstein; or, The Modern
/// Prometheus* (OL450063W) lists etext 84 — so the identifier rule alone left
/// them unbound, and their book pages, which ask for `openlibraryid:OL…W`,
/// empty. This is the weaker proof the trust model already has a grade for:
/// the caller stamps `anchorMethod=title`, the gateway records
/// `matchedBy/openlibrary:OL…W`, and meta-read ranks such an edition after an
/// identifier-bound one.
///
/// Same main title ([`main_title`]), an author with the same surname, and,
/// among those, the work with the most editions — Open Library holds many
/// small works titled *Frankenstein* by Mary Shelley (adaptations, annotated
/// and school editions), and the canonical one is the one most published.
/// `None` when nothing matches or two works tie.
pub async fn title_work(
&self,
title: &str,
author: &str,
deadline: Duration,
) -> Result<Option<String>, GatewayError> {
let (want, surname) = (main_title(title), surname(author));
if want.is_empty() || surname.is_empty() {
return Ok(None);
}
let key = format!("openlibrary:titlework:{want}:{surname}");
if let Some(c) = self.cache_get(&key) {
let work = c.body.get("work").and_then(Value::as_str).map(str::to_string);
let ttl = if work.is_some() {
BINDING_HIT_TTL_SECS
} else {
BINDING_MISS_TTL_SECS
};
if now_secs().saturating_sub(c.at) <= ttl {
return Ok(work);
}
}
let body = self
.get(
"/search.json",
&[
("title", want.clone()),
("author", surname.clone()),
("fields", "key,title,author_name,edition_count".into()),
("limit", "20".into()),
],
deadline,
)
.await?;
let work = best_title_work(&body, &want, &surname);
self.cache_put(&key, &serde_json::json!({ "work": work }));
Ok(work)
}
}
/// What Open Library records about one Gutenberg etext. See
/// [`OpenLibraryClient::gutenberg_etext`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Etext {
/// Exactly one work lists it: an identifier binding, graded Anchored.
One(String),
/// Several works list it: an unmerged duplicate, so no binding at all.
Several,
/// No work lists it: the one case a title binding may fill.
Unlisted,
}
impl Etext {
fn to_cached(&self) -> Value {
match self {
Etext::One(work) => serde_json::json!({ "etext": "one", "work": work }),
Etext::Several => serde_json::json!({ "etext": "several" }),
Etext::Unlisted => serde_json::json!({ "etext": "unlisted" }),
}
}
fn from_cached(v: &Value) -> Option<Self> {
match v.get("etext")?.as_str()? {
"one" => Some(Etext::One(v.get("work")?.as_str()?.to_string())),
"several" => Some(Etext::Several),
"unlisted" => Some(Etext::Unlisted),
_ => None,
}
}
}
fn etext_of(body: &Value, gutenberg_id: u64) -> Etext {
if let Some(work) = single_gutenberg_work(body, gutenberg_id) {
return Etext::One(work);
}
let docs = parse_docs(body.get("docs"));
let total = body
.get("numFound")
.or_else(|| body.get("num_found"))
.and_then(Value::as_u64)
.unwrap_or(docs.len() as u64);
if total == 0 && docs.is_empty() {
Etext::Unlisted
} else {
Etext::Several
}
}
/// The part of a title two catalogues agree on: before any subtitle (`;`, `:`,
/// a bracket, or `, or`), lower-cased, letters and digits only. Gutenberg's
/// *Frankenstein; Or, The Modern Prometheus* and Open Library's *Frankenstein;
/// or, The Modern Prometheus* are both `frankenstein`.
pub fn main_title(raw: &str) -> String {
let lower = raw.to_lowercase();
let head = lower
.split([';', ':', '(', '['])
.next()
.unwrap_or_default();
let head = head.find(", or").map_or(head, |i| &head[..i]);
head.split(|c: char| !c.is_alphanumeric())
.filter(|w| !w.is_empty())
.collect::<Vec<_>>()
.join(" ")
}
/// The surname a name ends on, lower-cased, letters and digits only: Gutendex
/// writes `Shelley, Mary Wollstonecraft`, Open Library `Mary Shelley`; both give
/// `shelley`.
pub fn surname(raw: &str) -> String {
let raw = raw.trim();
let last = match raw.split_once(',') {
Some((last, _)) => last,
None => raw.rsplit(char::is_whitespace).next().unwrap_or_default(),
};
last.chars()
.filter(|c| c.is_alphanumeric())
.collect::<String>()
.to_lowercase()
}
/// Does `name` carry `surname`? Compared with the separators removed, so a
/// compound surname (`Saint-Exupéry`) matches however it is spaced.
fn author_has_surname(name: &str, surname: &str) -> bool {
let flat = name
.chars()
.filter(|c| c.is_alphanumeric())
.collect::<String>()
.to_lowercase();
!surname.is_empty() && flat.contains(surname)
}
/// One document of the title search, with the edition count the choice rests on.
#[derive(Debug, Deserialize)]
struct TitleDoc {
#[serde(default)]
key: String,
#[serde(default)]
title: Option<String>,
#[serde(default)]
author_name: Vec<String>,
#[serde(default)]
edition_count: Option<u64>,
}
/// Decide a title binding from a `title=…&author=…` response. See
/// [`OpenLibraryClient::title_work`].
fn best_title_work(body: &Value, want: &str, surname: &str) -> Option<String> {
let docs: Vec<TitleDoc> = body
.get("docs")
.and_then(Value::as_array)
.map(|a| {
a.iter()
.filter_map(|d| serde_json::from_value::<TitleDoc>(d.clone()).ok())
.collect()
})
.unwrap_or_default();
let mut best: Option<(u64, String)> = None;
let mut tie = false;
for d in docs {
let Some(id) = d.key.strip_prefix("/works/").filter(|id| is_work_id(id)) else {
continue;
};
if main_title(d.title.as_deref().unwrap_or_default()) != want {
continue;
}
if !d.author_name.iter().any(|a| author_has_surname(a, surname)) {
continue;
}
let n = d.edition_count.unwrap_or(0);
let top = best.as_ref().map(|(m, _)| *m);
if top.is_none_or(|m| n > m) {
best = Some((n, id.to_string()));
tie = false;
} else if top == Some(n) {
tie = true;
}
}
if tie {
None
} else {
best.map(|(_, id)| id)
}
}
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
@@ -912,70 +698,6 @@ fn clean_description(raw: &str) -> Option<String> {
mod tests {
use super::*;
#[test]
fn a_main_title_drops_the_subtitle_case_and_punctuation() {
assert_eq!(main_title("Frankenstein; Or, The Modern Prometheus"), "frankenstein");
assert_eq!(main_title("Frankenstein; or, The Modern Prometheus"), "frankenstein");
assert_eq!(main_title("Frankenstein, or the Modern Prometheus"), "frankenstein");
assert_eq!(main_title("Frankenstein (Oxford Playscripts)"), "frankenstein");
assert_eq!(main_title("Alice's Adventures in Wonderland"), "alice s adventures in wonderland");
assert_eq!(main_title(""), "");
}
#[test]
fn a_surname_reads_both_name_orders() {
assert_eq!(surname("Shelley, Mary Wollstonecraft"), "shelley");
assert_eq!(surname("Mary Shelley"), "shelley");
assert_eq!(surname(" Austen, Jane "), "austen");
assert_eq!(surname("Saint-Exupéry, Antoine de"), "saintexupéry");
assert!(author_has_surname("Antoine de Saint-Exupéry", "saintexupéry"));
assert!(!author_has_surname("Jason Cobley", "shelley"));
}
fn title_doc(key: &str, title: &str, author: &str, editions: u64) -> Value {
serde_json::json!({ "key": key, "title": title, "author_name": [author], "edition_count": editions })
}
#[test]
fn the_title_binding_takes_the_most_published_work_of_that_title_and_author() {
let body = serde_json::json!({ "docs": [
title_doc("/works/OL26860869W", "Frankenstein", "Mary Shelley", 84),
title_doc("/works/OL450063W", "Frankenstein; or, The Modern Prometheus", "Mary Shelley", 2187),
title_doc("/works/OL25595002W", "Mary Shelley's Frankenstein", "Mary Shelley", 9000),
title_doc("/works/OL15443537W", "Frankenstein", "Dean Koontz", 5000),
]});
// The 9000-edition work has another title, the 5000 one another author.
assert_eq!(best_title_work(&body, "frankenstein", "shelley").as_deref(), Some("OL450063W"));
}
#[test]
fn a_tie_or_no_match_binds_nothing() {
let tie = serde_json::json!({ "docs": [
title_doc("/works/OL1W", "Frankenstein", "Mary Shelley", 10),
title_doc("/works/OL2W", "Frankenstein", "Mary Shelley", 10),
]});
assert_eq!(best_title_work(&tie, "frankenstein", "shelley"), None);
let other = serde_json::json!({ "docs": [title_doc("/works/OL3W", "Dracula", "Bram Stoker", 99)] });
assert_eq!(best_title_work(&other, "frankenstein", "shelley"), None);
let not_a_work = serde_json::json!({ "docs": [title_doc("/books/OL4M", "Frankenstein", "Mary Shelley", 99)] });
assert_eq!(best_title_work(&not_a_work, "frankenstein", "shelley"), None);
}
#[test]
fn an_etext_is_one_several_or_unlisted() {
let one = serde_json::json!({ "numFound": 1, "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }] });
assert_eq!(etext_of(&one, 1342), Etext::One("OL66554W".into()));
let several = serde_json::json!({ "numFound": 2, "docs": [
{ "key": "/works/OL450063W", "id_project_gutenberg": ["84"] },
{ "key": "/works/OL18388W", "id_project_gutenberg": ["84"] }
]});
assert_eq!(etext_of(&several, 84), Etext::Several);
assert_eq!(etext_of(&serde_json::json!({ "numFound": 0, "docs": [] }), 84), Etext::Unlisted);
for e in [Etext::One("OL1W".into()), Etext::Several, Etext::Unlisted] {
assert_eq!(Etext::from_cached(&e.to_cached()), Some(e));
}
}
#[test]
fn work_ids_are_works_only() {
assert!(is_work_id("OL66554W"));