Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6a9ab9e45 | ||
|
|
02bff131ff | ||
|
|
560effe226 | ||
|
|
e14797e57b | ||
|
|
efba2f2825 | ||
|
|
d177e7b710 | ||
|
|
6bfe36edb5 |
@@ -2,10 +2,10 @@
|
|||||||
# folder exists, and GitHub never reads it.
|
# folder exists, and GitHub never reads it.
|
||||||
#
|
#
|
||||||
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
||||||
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
# push ipcr-push-85-17-246-67.ip.nsl.sh/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
||||||
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
||||||
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
||||||
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
# The push goes to ipcr-push-85-17-246-67.ip.nsl.sh, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
||||||
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
||||||
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
||||||
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
||||||
@@ -27,7 +27,7 @@ on:
|
|||||||
tags: ['v*']
|
tags: ['v*']
|
||||||
|
|
||||||
env:
|
env:
|
||||||
REGISTRY: ipcr-push-85-17-246-67.nip.io
|
REGISTRY: ipcr-push-85-17-246-67.ip.nsl.sh
|
||||||
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
@@ -71,6 +71,9 @@ jobs:
|
|||||||
tags: |
|
tags: |
|
||||||
type=ref,event=branch
|
type=ref,event=branch
|
||||||
type=semver,pattern={{version}}
|
type=semver,pattern={{version}}
|
||||||
|
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||||
|
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||||
|
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
type=semver,pattern={{major}}
|
type=semver,pattern={{major}}
|
||||||
type=sha,format=short
|
type=sha,format=short
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# Runs the feeder's tests on every push. The repo had only the image workflow,
|
||||||
|
# so nothing checked a change before it shipped.
|
||||||
|
|
||||||
|
name: Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["**"]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
rust:
|
||||||
|
name: cargo test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# The job ran out of memory linking the test binaries (ld killed with
|
||||||
|
# signal 9, run 705 twice): no debug info, and at most two links at once.
|
||||||
|
# What is tested does not change.
|
||||||
|
env:
|
||||||
|
CARGO_PROFILE_DEV_DEBUG: "0"
|
||||||
|
CARGO_BUILD_JOBS: "2"
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
# rust-toolchain.toml pins the channel (with clippy); rustup installs it on
|
||||||
|
# first use, so the action only has to provide rustup itself.
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
components: clippy
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: cargo test
|
||||||
|
run: cargo test --locked
|
||||||
|
- name: cargo clippy
|
||||||
|
run: cargo clippy --all-targets -- -D warnings
|
||||||
@@ -43,6 +43,9 @@ jobs:
|
|||||||
tags: |
|
tags: |
|
||||||
type=ref,event=branch
|
type=ref,event=branch
|
||||||
type=semver,pattern={{version}}
|
type=semver,pattern={{version}}
|
||||||
|
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||||
|
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||||
|
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
type=semver,pattern={{major}}
|
type=semver,pattern={{major}}
|
||||||
type=sha,format=short
|
type=sha,format=short
|
||||||
|
|||||||
Generated
+1
-1
@@ -792,7 +792,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "meta-feeder-sdk"
|
name = "meta-feeder-sdk"
|
||||||
version = "1.7.0"
|
version = "1.7.0"
|
||||||
source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
source = "git+https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ path = "src/main.rs"
|
|||||||
# vendored copy this used to point at had already drifted behind the shared SDK
|
# vendored copy this used to point at had already drifted behind the shared SDK
|
||||||
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
||||||
# way; see meta-feeder-anilist's manifest.
|
# way; see meta-feeder-anilist's manifest.
|
||||||
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
|
meta-feeder-sdk = { git = "https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk", tag = "v1.7.0" }
|
||||||
|
|
||||||
tokio = { version = "1.40", features = ["full"] }
|
tokio = { version = "1.40", features = ["full"] }
|
||||||
axum = { version = "0.7", features = ["json"] }
|
axum = { version = "0.7", features = ["json"] }
|
||||||
|
|||||||
@@ -32,7 +32,37 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use crate::openlibrary_client::{self, OpenLibraryClient};
|
use crate::openlibrary::first_filter;
|
||||||
|
use crate::openlibrary_client::{self, Etext, OpenLibraryClient};
|
||||||
|
|
||||||
|
/// An edition's Open Library work, and how it was found.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct Binding {
|
||||||
|
work: String,
|
||||||
|
/// Bound by title and author ([`OpenLibraryClient::title_work`]) rather than
|
||||||
|
/// by an identifier Open Library records: graded Matched, not Anchored.
|
||||||
|
by_title: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One etext's binding: the identifier Open Library records, else — only for an
|
||||||
|
/// etext it lists under no work — the title match. Shared by the search path and
|
||||||
|
/// the redeem path so the two cannot bind the same book differently.
|
||||||
|
async fn bind(
|
||||||
|
client: &OpenLibraryClient,
|
||||||
|
id: u64,
|
||||||
|
title: &str,
|
||||||
|
author: &str,
|
||||||
|
deadline: std::time::Duration,
|
||||||
|
) -> Result<Option<Binding>, GatewayError> {
|
||||||
|
Ok(match client.gutenberg_etext(id, deadline).await? {
|
||||||
|
Etext::One(work) => Some(Binding { work, by_title: false }),
|
||||||
|
Etext::Several => None,
|
||||||
|
Etext::Unlisted => client
|
||||||
|
.title_work(title, author, deadline)
|
||||||
|
.await?
|
||||||
|
.map(|work| Binding { work, by_title: true }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
||||||
/// for tests + private mirrors.
|
/// for tests + private mirrors.
|
||||||
@@ -390,31 +420,60 @@ impl GutenbergPlugin {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The Gutendex search a work page's query stands for.
|
||||||
|
///
|
||||||
|
/// meta-read's book page asks for `openlibraryid:OL…W` with no free text.
|
||||||
|
/// That used to fall through to the catalogue browse — the 32 most
|
||||||
|
/// downloaded books — and the gateway kept those bound to the work, so a
|
||||||
|
/// book outside that list never showed an edition on its own page. Searching
|
||||||
|
/// for the work's main title and its first author's surname finds it
|
||||||
|
/// wherever it ranks; the gateway still keeps only the records bound to the
|
||||||
|
/// work. `None` (no work named, Open Library off or failing) keeps the browse.
|
||||||
|
async fn work_search_text(&self, query: &GatewayQuery) -> Option<String> {
|
||||||
|
let work = first_filter(query, "openlibraryid")?.to_ascii_uppercase();
|
||||||
|
if !openlibrary_client::is_work_id(&work) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let client = self.openlibrary.as_ref()?;
|
||||||
|
let lookup = client.doc_by_work_id(&work, openlibrary_client::BINDING_DEADLINE);
|
||||||
|
let doc = tokio::time::timeout(BINDING_TIMEOUT, lookup).await.ok()?.ok()??;
|
||||||
|
let title = openlibrary_client::main_title(doc.title.as_deref()?);
|
||||||
|
if title.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let author = doc.author_name.first().map(|a| openlibrary_client::surname(a)).unwrap_or_default();
|
||||||
|
Some(format!("{title} {author}").trim().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// Each result's Open Library work, through the Project Gutenberg id Open
|
/// Each result's Open Library work, through the Project Gutenberg id Open
|
||||||
/// Library records (reading-model §2.4). Aligned with the input slice;
|
/// Library records (reading-model §2.4), or, for an etext Open Library lists
|
||||||
/// `None` means no binding — none recorded, ambiguous, not configured, over
|
/// under no work, by title and author (`by_title`, graded Matched). Aligned
|
||||||
/// budget, or the lookup failed.
|
/// with the input slice; `None` means no binding — ambiguous, no title match,
|
||||||
|
/// not configured, over budget, or the lookup failed.
|
||||||
///
|
///
|
||||||
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
||||||
/// edition that is already readable. Every failure is left uncached, so the
|
/// edition that is already readable. Every failure is left uncached, so the
|
||||||
/// next query that surfaces the book tries again; the budget deadline is
|
/// next query that surfaces the book tries again; the budget deadline is
|
||||||
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
||||||
/// first handful of books and the cache fills the rest over later queries.
|
/// first handful of books and the cache fills the rest over later queries.
|
||||||
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<String>> {
|
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<Binding>> {
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
const BINDING_CONCURRENCY: usize = 4;
|
const BINDING_CONCURRENCY: usize = 4;
|
||||||
let Some(client) = self.openlibrary.as_ref() else {
|
let Some(client) = self.openlibrary.as_ref() else {
|
||||||
return vec![None; books.len()];
|
return vec![None; books.len()];
|
||||||
};
|
};
|
||||||
// Owned ids, not `books.iter().map(..)`: a borrowing closure inside the
|
// Owned values, not `books.iter().map(..)`: a borrowing closure inside the
|
||||||
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
||||||
// check ("FnOnce is not general enough").
|
// check ("FnOnce is not general enough").
|
||||||
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
let wanted: Vec<(u64, String, String)> = books
|
||||||
stream::iter(ids)
|
.iter()
|
||||||
.map(|id| async move {
|
.map(|b| (b.id, b.title.clone(), b.authors.first().map(|a| a.name.clone()).unwrap_or_default()))
|
||||||
let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE);
|
.collect();
|
||||||
|
stream::iter(wanted)
|
||||||
|
.map(|(id, title, author)| async move {
|
||||||
|
let lookup = bind(client, id, &title, &author, openlibrary_client::BINDING_DEADLINE);
|
||||||
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
||||||
Ok(Ok(work)) => work,
|
Ok(Ok(binding)) => binding,
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
debug!(
|
debug!(
|
||||||
target: "meta-share::gateway",
|
target: "meta-share::gateway",
|
||||||
@@ -507,7 +566,16 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// everything, already ordered by download count (identical to
|
// everything, already ordered by download count (identical to
|
||||||
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
||||||
// with whatever happens to contain punctuation.
|
// with whatever happens to contain punctuation.
|
||||||
let q = query.free_text.trim();
|
// A work page's query names its Open Library work and no text: search
|
||||||
|
// for that work's title and author instead of browsing. See
|
||||||
|
// `work_search_text`.
|
||||||
|
let free = query.free_text.trim();
|
||||||
|
let work_text = if free.is_empty() {
|
||||||
|
self.work_search_text(query).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let q = work_text.as_deref().unwrap_or(free);
|
||||||
let params: Vec<(&str, &str)> = if q.is_empty() {
|
let params: Vec<(&str, &str)> = if q.is_empty() {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
} else {
|
} else {
|
||||||
@@ -568,9 +636,9 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// Not user-facing, so the binding lookup may queue for the full search
|
// Not user-facing, so the binding lookup may queue for the full search
|
||||||
// deadline. A failure still leaves the record unbound, never fails the
|
// deadline. A failure still leaves the record unbound, never fails the
|
||||||
// compute.
|
// compute.
|
||||||
|
let author = book.authors.first().map(|a| a.name.clone()).unwrap_or_default();
|
||||||
let work = match self.openlibrary.as_ref() {
|
let work = match self.openlibrary.as_ref() {
|
||||||
Some(ol) => ol
|
Some(ol) => bind(ol, book.id, &book.title, &author, openlibrary_client::SEARCH_DEADLINE)
|
||||||
.gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE)
|
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -644,13 +712,13 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
|
|
||||||
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
||||||
///
|
///
|
||||||
/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]:
|
/// `openlibrary_work` is the binding from [`bind`]: the one work Open Library
|
||||||
/// `Some` only when Open Library records exactly one work for this etext.
|
/// records for this etext, or, when it records none, the title match.
|
||||||
fn into_discovery_record(
|
fn into_discovery_record(
|
||||||
book: GutendexBook,
|
book: GutendexBook,
|
||||||
cover_cid: Option<String>,
|
cover_cid: Option<String>,
|
||||||
bibrec: BTreeMap<String, String>,
|
bibrec: BTreeMap<String, String>,
|
||||||
openlibrary_work: Option<String>,
|
openlibrary_work: Option<Binding>,
|
||||||
) -> DiscoveryRecord {
|
) -> DiscoveryRecord {
|
||||||
let GutendexBook {
|
let GutendexBook {
|
||||||
id,
|
id,
|
||||||
@@ -803,16 +871,25 @@ fn into_discovery_record(
|
|||||||
let _ = download_count;
|
let _ = download_count;
|
||||||
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
||||||
|
|
||||||
// The work binding and its trust claim (reading-model §2.4). Written only on
|
// The work binding and its trust claim (reading-model §2.4). On an
|
||||||
// an identifier link Open Library itself records, so the claim is the plain
|
// identifier link Open Library itself records, the claim is the plain
|
||||||
// Anchored form: `anchored` + `anchorSource`, and deliberately NO
|
// Anchored form: `anchored` + `anchorSource`, and NO `anchorMethod` — that
|
||||||
// `anchorMethod` — that marks a title match, which this never is. The
|
// marks a title match. The
|
||||||
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
||||||
// strips the transient keys before storing.
|
// strips the transient keys before storing.
|
||||||
if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) {
|
//
|
||||||
fields.insert("openlibraryid".into(), work);
|
// A title binding (an etext Open Library lists under no work) carries the
|
||||||
|
// same claim plus `anchorMethod=title`: the gateway then records
|
||||||
|
// `matchedBy/openlibrary:OL…W` instead of `anchoredBy/`, and meta-read grades
|
||||||
|
// the edition Matched — listed on its work's page, ranked after any
|
||||||
|
// identifier-bound edition.
|
||||||
|
if let Some(binding) = openlibrary_work.filter(|b| openlibrary_client::is_work_id(&b.work)) {
|
||||||
|
fields.insert("openlibraryid".into(), binding.work);
|
||||||
fields.insert("anchored".into(), "true".into());
|
fields.insert("anchored".into(), "true".into());
|
||||||
fields.insert("anchorSource".into(), "openlibrary".into());
|
fields.insert("anchorSource".into(), "openlibrary".into());
|
||||||
|
if binding.by_title {
|
||||||
|
fields.insert("anchorMethod".into(), "title".into());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (k, v) in bibrec {
|
for (k, v) in bibrec {
|
||||||
@@ -1678,6 +1755,98 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open Library's answer to `title=…&author=…` for Frankenstein: several
|
||||||
|
/// works by Mary Shelley share the title; the most published is the novel.
|
||||||
|
fn frankenstein_by_title() -> Mock {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("title", "frankenstein"))
|
||||||
|
.and(query_param("author", "shelley"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 3, "docs": [
|
||||||
|
{ "key": "/works/OL26860869W", "title": "Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 84 },
|
||||||
|
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"], "edition_count": 2187 },
|
||||||
|
{ "key": "/works/OL25595002W", "title": "Mary Shelley's Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 90 }
|
||||||
|
]})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ⚠ Open Library lists etext 84 under no work, so the identifier rule alone
|
||||||
|
/// left the most read novel on Gutenberg unbound and its book page empty.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_etext_no_work_lists_is_bound_by_title_and_graded_matched() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(
|
||||||
|
1342,
|
||||||
|
serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }
|
||||||
|
]}),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
frankenstein_by_title().expect(1).mount(&server).await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("handle_query");
|
||||||
|
|
||||||
|
let pride = &records[0].fields;
|
||||||
|
assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W"));
|
||||||
|
assert!(!pride.contains_key("anchorMethod"), "an identifier binding stays Anchored");
|
||||||
|
|
||||||
|
let frankenstein = &records[1].fields;
|
||||||
|
assert_eq!(frankenstein.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
||||||
|
assert_eq!(frankenstein.get("anchored").map(String::as_str), Some("true"));
|
||||||
|
assert_eq!(frankenstein.get("anchorSource").map(String::as_str), Some("openlibrary"));
|
||||||
|
assert_eq!(frankenstein.get("anchorMethod").map(String::as_str), Some("title"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ⚠ A work page asks for its work and no text. That used to browse the 32
|
||||||
|
/// most downloaded books, so a book outside them never reached its own page.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_work_page_query_searches_for_the_works_title_and_author() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "key:/works/OL450063W"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"] }
|
||||||
|
]})))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.and(query_param("search", "frankenstein shelley"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(1342, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
frankenstein_by_title().mount(&server).await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let mut query = GatewayQuery::from_free_text("");
|
||||||
|
query.filters.insert("openlibraryid".into(), vec!["OL450063W".into()]);
|
||||||
|
query.filters.insert("contentKind".into(), vec!["book".into()]);
|
||||||
|
let records = plugin.handle_query(&query, 50).await.expect("handle_query");
|
||||||
|
let frankenstein = records.iter().find(|r| r.record_id == "84").expect("etext 84 is found");
|
||||||
|
assert_eq!(frankenstein.fields.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
||||||
|
assert_eq!(frankenstein.fields.get("anchorMethod").map(String::as_str), Some("title"));
|
||||||
|
}
|
||||||
|
|
||||||
/// ⚠ A binding failure never costs the record.
|
/// ⚠ A binding failure never costs the record.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ fn env_usize(key: &str, default: usize) -> usize {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// First non-empty value for `key`.
|
/// First non-empty value for `key`.
|
||||||
fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
pub(crate) fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
||||||
q.filters
|
q.filters
|
||||||
.get(key)
|
.get(key)
|
||||||
.and_then(|v| v.first())
|
.and_then(|v| v.first())
|
||||||
@@ -169,33 +169,14 @@ fn foreign_identity_filter(q: &GatewayQuery) -> Option<&str> {
|
|||||||
/// what passes, permanently. A card that lacks another filter's key does not
|
/// what passes, permanently. A card that lacks another filter's key does not
|
||||||
/// match it, which is the truth.
|
/// match it, which is the truth.
|
||||||
pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option<DiscoveryRecord> {
|
pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option<DiscoveryRecord> {
|
||||||
to_record_with(doc, Some(description), query, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// [`to_record`] with the quality bar as a choice.
|
|
||||||
///
|
|
||||||
/// `lenient = false` is the bar every search and browse row keeps: a cover **and**
|
|
||||||
/// a description, or no card. `lenient = true` is for a lookup that names one work
|
|
||||||
/// by `openlibraryid` (a work page): the work is already known to be wanted, so a
|
|
||||||
/// title is enough. Its cover may come from the work's ISBNs when the work has none
|
|
||||||
/// of its own, and a missing description is simply left out.
|
|
||||||
pub fn to_record_with(
|
|
||||||
doc: &SearchDoc,
|
|
||||||
description: Option<&str>,
|
|
||||||
query: &GatewayQuery,
|
|
||||||
lenient: bool,
|
|
||||||
) -> Option<DiscoveryRecord> {
|
|
||||||
let id = doc.work_id()?;
|
let id = doc.work_id()?;
|
||||||
let cid = card_cid(id)?;
|
let cid = card_cid(id)?;
|
||||||
let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?;
|
let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?;
|
||||||
let description = description.map(str::trim).filter(|d| !d.is_empty());
|
let poster_url = doc.cover_url()?;
|
||||||
let poster_url = if lenient {
|
let description = description.trim();
|
||||||
doc.cover_url().or_else(|| doc.isbn_cover_url())
|
if description.is_empty() {
|
||||||
} else {
|
return None;
|
||||||
// Strict: both are required.
|
}
|
||||||
description?;
|
|
||||||
Some(doc.cover_url()?)
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
||||||
|
|
||||||
@@ -219,9 +200,7 @@ pub fn to_record_with(
|
|||||||
|
|
||||||
// `description/{lang3}`, never a flat `description` — the flat key is indexed
|
// `description/{lang3}`, never a flat `description` — the flat key is indexed
|
||||||
// but never written (METADATA_KEYS.md §14.6).
|
// but never written (METADATA_KEYS.md §14.6).
|
||||||
if let Some(d) = description {
|
fields.insert("description/eng".into(), description.to_string());
|
||||||
fields.insert("description/eng".into(), d.to_string());
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
let authors: Vec<&str> = doc
|
let authors: Vec<&str> = doc
|
||||||
@@ -245,9 +224,7 @@ pub fn to_record_with(
|
|||||||
|
|
||||||
// ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS
|
// ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS
|
||||||
// path fetches it, seeds the bytes and writes the content cid back.
|
// path fetches it, seeds the bytes and writes the content cid back.
|
||||||
if let Some(url) = poster_url {
|
fields.insert("poster_url".into(), poster_url);
|
||||||
fields.insert("poster_url".into(), url);
|
|
||||||
}
|
|
||||||
|
|
||||||
fields.insert(
|
fields.insert(
|
||||||
"sourceUrl".into(),
|
"sourceUrl".into(),
|
||||||
@@ -280,14 +257,13 @@ async fn project(
|
|||||||
query: &GatewayQuery,
|
query: &GatewayQuery,
|
||||||
n: usize,
|
n: usize,
|
||||||
deadline: Duration,
|
deadline: Duration,
|
||||||
lenient: bool,
|
|
||||||
) -> Vec<DiscoveryRecord> {
|
) -> Vec<DiscoveryRecord> {
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
let candidates: Vec<SearchDoc> = docs
|
let candidates: Vec<SearchDoc> = docs
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|d| {
|
.filter(|d| {
|
||||||
d.work_id().is_some()
|
d.work_id().is_some()
|
||||||
&& (lenient || d.cover_url().is_some())
|
&& d.cover_url().is_some()
|
||||||
&& d.title.as_deref().is_some_and(|t| !t.trim().is_empty())
|
&& d.title.as_deref().is_some_and(|t| !t.trim().is_empty())
|
||||||
})
|
})
|
||||||
.filter(|d| seen.insert(d.key.clone()))
|
.filter(|d| seen.insert(d.key.clone()))
|
||||||
@@ -310,7 +286,7 @@ async fn project(
|
|||||||
debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped");
|
debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped");
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
to_record_with(&doc, w.description.as_deref(), query, lenient)
|
to_record(&doc, w.description.as_deref()?, query)
|
||||||
}
|
}
|
||||||
Ok(None) => None,
|
Ok(None) => None,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -461,9 +437,7 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else {
|
let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
};
|
};
|
||||||
// Lenient: an explicit work lookup keeps the card even without a
|
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE).await);
|
||||||
// description, and finds a cover by ISBN when the work has none.
|
|
||||||
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE, true).await);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A curated row.
|
// A curated row.
|
||||||
@@ -481,7 +455,7 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return Ok(project(client, docs, query, n, BROWSE_DEADLINE, false).await);
|
return Ok(project(client, docs, query, n, BROWSE_DEADLINE).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Free text.
|
// Free text.
|
||||||
@@ -496,7 +470,7 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
let docs = client
|
let docs = client
|
||||||
.search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE)
|
.search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(project(client, docs, query, n, SEARCH_DEADLINE, false).await)
|
Ok(project(client, docs, query, n, SEARCH_DEADLINE).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A card has no bytes: the record IS the payload.
|
/// A card has no bytes: the record IS the payload.
|
||||||
@@ -515,8 +489,7 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
// different work (a merge) is not this cid's record.
|
// different work (a merge) is not this cid's record.
|
||||||
let record = match self.client.as_ref() {
|
let record = match self.client.as_ref() {
|
||||||
Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await {
|
Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await {
|
||||||
// Lenient, like the `openlibraryid` lookup that minted this card.
|
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE)
|
||||||
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE, true)
|
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find(|r| r.record_id == id),
|
.find(|r| r.record_id == id),
|
||||||
@@ -572,7 +545,6 @@ mod tests {
|
|||||||
cover_i: Some(14348537),
|
cover_i: Some(14348537),
|
||||||
first_publish_year: Some(1813),
|
first_publish_year: Some(1813),
|
||||||
id_project_gutenberg: vec![],
|
id_project_gutenberg: vec![],
|
||||||
isbn: vec![],
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -679,95 +651,6 @@ mod tests {
|
|||||||
assert!(to_record(&edition, "A novel.", &q("", &[])).is_none());
|
assert!(to_record(&edition, "A novel.", &q("", &[])).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_lenient_card_needs_only_a_title_and_finds_a_cover_by_isbn() {
|
|
||||||
let mut d = pride();
|
|
||||||
d.cover_i = None;
|
|
||||||
d.isbn = vec!["1955252092".into(), "9781955252096".into()];
|
|
||||||
// Strict still refuses it, whatever the description.
|
|
||||||
assert!(to_record_with(&d, Some("A novel."), &q("", &[]), false).is_none());
|
|
||||||
// Lenient mints it with the ISBN-13 cover and no description at all.
|
|
||||||
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
|
|
||||||
assert_eq!(
|
|
||||||
r.fields["poster_url"],
|
|
||||||
"https://covers.openlibrary.org/b/isbn/9781955252096-L.jpg?default=false"
|
|
||||||
);
|
|
||||||
assert!(!r.fields.keys().any(|k| k.starts_with("description")));
|
|
||||||
assert_eq!(r.fields["openlibraryid"], "OL66554W");
|
|
||||||
assert_eq!(r.fields["fileType"], "card");
|
|
||||||
// Same address as the strict card for the same work.
|
|
||||||
let strict = to_record(&pride(), "A novel.", &q("", &[])).unwrap();
|
|
||||||
let key = |r: &DiscoveryRecord| r.fields.keys().find(|k| k.starts_with("cids/")).cloned();
|
|
||||||
assert_eq!(key(&r), key(&strict));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_lenient_card_with_no_cover_anywhere_has_no_poster_rather_than_no_card() {
|
|
||||||
let mut d = pride();
|
|
||||||
d.cover_i = None;
|
|
||||||
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
|
|
||||||
assert!(!r.fields.contains_key("poster_url"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn isbn_cover_prefers_an_isbn13_and_ignores_junk() {
|
|
||||||
let mut d = SearchDoc::default();
|
|
||||||
d.isbn = vec!["not-an-isbn".into(), "0306406152".into(), "9780306406157".into()];
|
|
||||||
assert_eq!(
|
|
||||||
d.isbn_cover_url().as_deref(),
|
|
||||||
Some("https://covers.openlibrary.org/b/isbn/9780306406157-L.jpg?default=false")
|
|
||||||
);
|
|
||||||
d.isbn = vec!["0306406152".into()];
|
|
||||||
assert!(d.isbn_cover_url().unwrap().contains("0306406152"));
|
|
||||||
d.isbn = vec!["x".into()];
|
|
||||||
assert_eq!(d.isbn_cover_url(), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_named_work_without_cover_or_description_still_gets_its_card_but_a_search_does_not() {
|
|
||||||
let server = MockServer::start().await;
|
|
||||||
let doc = json!({
|
|
||||||
"key": "/works/OL39284292W",
|
|
||||||
"title": "Mother of Learning",
|
|
||||||
"author_name": ["nobody103"],
|
|
||||||
"first_publish_year": 2021,
|
|
||||||
"language": ["eng"],
|
|
||||||
"isbn": ["9781955252096"]
|
|
||||||
});
|
|
||||||
// The lookup by id and the free-text search both find the same index doc.
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/search.json"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"numFound": 1, "docs": [doc]})))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/works/OL39284292W.json"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
|
||||||
"key": "/works/OL39284292W", "title": "Mother of Learning"
|
|
||||||
})))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
let (p, _dir) = plugin_against(&server).await;
|
|
||||||
|
|
||||||
let by_id = p
|
|
||||||
.handle_query(
|
|
||||||
&q("", &[("openlibraryid", "OL39284292W"), ("contentKind", "book")]),
|
|
||||||
10,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(by_id.len(), 1, "the work page lookup yields a card");
|
|
||||||
assert_eq!(by_id[0].fields["title"], "Mother of Learning");
|
|
||||||
assert!(by_id[0].fields["poster_url"].contains("/b/isbn/9781955252096-L.jpg"));
|
|
||||||
assert!(!by_id[0].fields.keys().any(|k| k.starts_with("description")));
|
|
||||||
|
|
||||||
let by_search = p
|
|
||||||
.handle_query(&q("mother of learning", &[("fileType", "card"), ("contentKind", "book")]), 10)
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
assert!(by_search.is_empty(), "search rows keep the strict bar");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn echoes_browse_markers_but_no_other_filter() {
|
fn echoes_browse_markers_but_no_other_filter() {
|
||||||
let r = to_record(
|
let r = to_record(
|
||||||
|
|||||||
@@ -30,9 +30,6 @@ pub const DEFAULT_API_BASE: &str = "https://openlibrary.org";
|
|||||||
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
|
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
|
||||||
/// `poster_url` names, so one size is published, not three.
|
/// `poster_url` names, so one size is published, not three.
|
||||||
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
|
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
|
||||||
/// Cover by ISBN. Open Library often holds an *edition's* cover for a work whose
|
|
||||||
/// own `covers` list is empty.
|
|
||||||
const COVERS_ISBN_BASE: &str = "https://covers.openlibrary.org/b/isbn";
|
|
||||||
|
|
||||||
/// Config key (and env seed) for the operator contact sent in the User-Agent.
|
/// Config key (and env seed) for the operator contact sent in the User-Agent.
|
||||||
pub const CONTACT_KEY: &str = "contact";
|
pub const CONTACT_KEY: &str = "contact";
|
||||||
@@ -61,9 +58,6 @@ const RETRY_BACKOFF_MS: u64 = 250;
|
|||||||
/// The field selection for every doc-shaped request, written once so a card
|
/// The field selection for every doc-shaped request, written once so a card
|
||||||
/// resolved by search and the same card resolved by id are byte-identical.
|
/// resolved by search and the same card resolved by id are byte-identical.
|
||||||
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
|
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
|
||||||
/// A lookup of one named work also asks for its ISBNs, so a cover can be found by
|
|
||||||
/// ISBN. Not on the shared list: a search row would carry hundreds of ISBNs per doc.
|
|
||||||
const DOC_BY_ID_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language,isbn";
|
|
||||||
|
|
||||||
/// Cache lifetimes.
|
/// Cache lifetimes.
|
||||||
///
|
///
|
||||||
@@ -205,10 +199,6 @@ pub struct SearchDoc {
|
|||||||
pub first_publish_year: Option<i32>,
|
pub first_publish_year: Option<i32>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub id_project_gutenberg: Vec<String>,
|
pub id_project_gutenberg: Vec<String>,
|
||||||
/// ISBNs of the work's editions. Only requested for an `openlibraryid`
|
|
||||||
/// lookup (see [`DOC_BY_ID_FIELDS`]); empty everywhere else.
|
|
||||||
#[serde(default)]
|
|
||||||
pub isbn: Vec<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SearchDoc {
|
impl SearchDoc {
|
||||||
@@ -224,22 +214,6 @@ impl SearchDoc {
|
|||||||
.filter(|c| *c > 0)
|
.filter(|c| *c > 0)
|
||||||
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
|
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A cover looked up by one of the work's ISBNs (an ISBN-13 when there is one),
|
|
||||||
/// for a work with no cover of its own.
|
|
||||||
///
|
|
||||||
/// `?default=false` makes Open Library answer 404 when it has no image for the
|
|
||||||
/// ISBN, instead of a 1×1 placeholder that would be seeded as the poster.
|
|
||||||
pub fn isbn_cover_url(&self) -> Option<String> {
|
|
||||||
let digits_only = |s: &&String| s.chars().all(|c| c.is_ascii_digit());
|
|
||||||
let isbn = self
|
|
||||||
.isbn
|
|
||||||
.iter()
|
|
||||||
.filter(digits_only)
|
|
||||||
.find(|s| s.len() == 13)
|
|
||||||
.or_else(|| self.isbn.iter().filter(digits_only).find(|s| s.len() == 10))?;
|
|
||||||
Some(format!("{COVERS_ISBN_BASE}/{isbn}-L.jpg?default=false"))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What a work JSON contributes to a card.
|
/// What a work JSON contributes to a card.
|
||||||
@@ -466,13 +440,12 @@ impl OpenLibraryClient {
|
|||||||
) -> Result<Option<SearchDoc>, GatewayError> {
|
) -> Result<Option<SearchDoc>, GatewayError> {
|
||||||
let body = self
|
let body = self
|
||||||
.get_cached(
|
.get_cached(
|
||||||
// `key2`: entries cached before ISBNs were requested lack them.
|
&format!("openlibrary:key:{id}"),
|
||||||
&format!("openlibrary:key2:{id}"),
|
|
||||||
WORK_TTL_SECS,
|
WORK_TTL_SECS,
|
||||||
"/search.json",
|
"/search.json",
|
||||||
&[
|
&[
|
||||||
("q", format!("key:/works/{id}")),
|
("q", format!("key:/works/{id}")),
|
||||||
("fields", DOC_BY_ID_FIELDS.into()),
|
("fields", SEARCH_FIELDS.into()),
|
||||||
("limit", "1".into()),
|
("limit", "1".into()),
|
||||||
],
|
],
|
||||||
deadline,
|
deadline,
|
||||||
@@ -520,7 +493,6 @@ impl OpenLibraryClient {
|
|||||||
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
|
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
|
||||||
first_publish_year: None,
|
first_publish_year: None,
|
||||||
id_project_gutenberg: Vec::new(),
|
id_project_gutenberg: Vec::new(),
|
||||||
isbn: Vec::new(),
|
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -642,6 +614,248 @@ impl OpenLibraryClient {
|
|||||||
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
Ok(work)
|
Ok(work)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What Open Library records about an etext: the one work that lists it,
|
||||||
|
/// several works, or none.
|
||||||
|
///
|
||||||
|
/// Same lookup as [`Self::gutenberg_work`], but it keeps the difference
|
||||||
|
/// between "several" and "none", which that one folds into `None`: only an
|
||||||
|
/// etext no work lists may be bound by title ([`Self::title_work`]). Several
|
||||||
|
/// works listing one etext is an unmerged duplicate, and guessing between
|
||||||
|
/// them is exactly what the identifier rule refuses.
|
||||||
|
pub async fn gutenberg_etext(
|
||||||
|
&self,
|
||||||
|
gutenberg_id: u64,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Etext, GatewayError> {
|
||||||
|
let key = format!("openlibrary:etext:{gutenberg_id}");
|
||||||
|
if let Some(c) = self.cache_get(&key) {
|
||||||
|
if let Some(etext) = Etext::from_cached(&c.body) {
|
||||||
|
let ttl = if matches!(etext, Etext::One(_)) {
|
||||||
|
BINDING_HIT_TTL_SECS
|
||||||
|
} else {
|
||||||
|
BINDING_MISS_TTL_SECS
|
||||||
|
};
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl {
|
||||||
|
return Ok(etext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self
|
||||||
|
.get(
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("id_project_gutenberg:{gutenberg_id}")),
|
||||||
|
("fields", "key,id_project_gutenberg".into()),
|
||||||
|
("limit", "2".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let etext = etext_of(&body, gutenberg_id);
|
||||||
|
self.cache_put(&key, &etext.to_cached());
|
||||||
|
Ok(etext)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The work a book is bound to by **title and author**, for an etext Open
|
||||||
|
/// Library lists under no work.
|
||||||
|
///
|
||||||
|
/// Open Library's `id_project_gutenberg` links are patchy even for the most
|
||||||
|
/// read books — none of the editions of *Frankenstein; or, The Modern
|
||||||
|
/// Prometheus* (OL450063W) lists etext 84 — so the identifier rule alone left
|
||||||
|
/// them unbound, and their book pages, which ask for `openlibraryid:OL…W`,
|
||||||
|
/// empty. This is the weaker proof the trust model already has a grade for:
|
||||||
|
/// the caller stamps `anchorMethod=title`, the gateway records
|
||||||
|
/// `matchedBy/openlibrary:OL…W`, and meta-read ranks such an edition after an
|
||||||
|
/// identifier-bound one.
|
||||||
|
///
|
||||||
|
/// Same main title ([`main_title`]), an author with the same surname, and,
|
||||||
|
/// among those, the work with the most editions — Open Library holds many
|
||||||
|
/// small works titled *Frankenstein* by Mary Shelley (adaptations, annotated
|
||||||
|
/// and school editions), and the canonical one is the one most published.
|
||||||
|
/// `None` when nothing matches or two works tie.
|
||||||
|
pub async fn title_work(
|
||||||
|
&self,
|
||||||
|
title: &str,
|
||||||
|
author: &str,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Option<String>, GatewayError> {
|
||||||
|
let (want, surname) = (main_title(title), surname(author));
|
||||||
|
if want.is_empty() || surname.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let key = format!("openlibrary:titlework:{want}:{surname}");
|
||||||
|
if let Some(c) = self.cache_get(&key) {
|
||||||
|
let work = c.body.get("work").and_then(Value::as_str).map(str::to_string);
|
||||||
|
let ttl = if work.is_some() {
|
||||||
|
BINDING_HIT_TTL_SECS
|
||||||
|
} else {
|
||||||
|
BINDING_MISS_TTL_SECS
|
||||||
|
};
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl {
|
||||||
|
return Ok(work);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self
|
||||||
|
.get(
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("title", want.clone()),
|
||||||
|
("author", surname.clone()),
|
||||||
|
("fields", "key,title,author_name,edition_count".into()),
|
||||||
|
("limit", "20".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let work = best_title_work(&body, &want, &surname);
|
||||||
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
|
Ok(work)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What Open Library records about one Gutenberg etext. See
|
||||||
|
/// [`OpenLibraryClient::gutenberg_etext`].
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Etext {
|
||||||
|
/// Exactly one work lists it: an identifier binding, graded Anchored.
|
||||||
|
One(String),
|
||||||
|
/// Several works list it: an unmerged duplicate, so no binding at all.
|
||||||
|
Several,
|
||||||
|
/// No work lists it: the one case a title binding may fill.
|
||||||
|
Unlisted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Etext {
|
||||||
|
fn to_cached(&self) -> Value {
|
||||||
|
match self {
|
||||||
|
Etext::One(work) => serde_json::json!({ "etext": "one", "work": work }),
|
||||||
|
Etext::Several => serde_json::json!({ "etext": "several" }),
|
||||||
|
Etext::Unlisted => serde_json::json!({ "etext": "unlisted" }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_cached(v: &Value) -> Option<Self> {
|
||||||
|
match v.get("etext")?.as_str()? {
|
||||||
|
"one" => Some(Etext::One(v.get("work")?.as_str()?.to_string())),
|
||||||
|
"several" => Some(Etext::Several),
|
||||||
|
"unlisted" => Some(Etext::Unlisted),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn etext_of(body: &Value, gutenberg_id: u64) -> Etext {
|
||||||
|
if let Some(work) = single_gutenberg_work(body, gutenberg_id) {
|
||||||
|
return Etext::One(work);
|
||||||
|
}
|
||||||
|
let docs = parse_docs(body.get("docs"));
|
||||||
|
let total = body
|
||||||
|
.get("numFound")
|
||||||
|
.or_else(|| body.get("num_found"))
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(docs.len() as u64);
|
||||||
|
if total == 0 && docs.is_empty() {
|
||||||
|
Etext::Unlisted
|
||||||
|
} else {
|
||||||
|
Etext::Several
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The part of a title two catalogues agree on: before any subtitle (`;`, `:`,
|
||||||
|
/// a bracket, or `, or`), lower-cased, letters and digits only. Gutenberg's
|
||||||
|
/// *Frankenstein; Or, The Modern Prometheus* and Open Library's *Frankenstein;
|
||||||
|
/// or, The Modern Prometheus* are both `frankenstein`.
|
||||||
|
pub fn main_title(raw: &str) -> String {
|
||||||
|
let lower = raw.to_lowercase();
|
||||||
|
let head = lower
|
||||||
|
.split([';', ':', '(', '['])
|
||||||
|
.next()
|
||||||
|
.unwrap_or_default();
|
||||||
|
let head = head.find(", or").map_or(head, |i| &head[..i]);
|
||||||
|
head.split(|c: char| !c.is_alphanumeric())
|
||||||
|
.filter(|w| !w.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The surname a name ends on, lower-cased, letters and digits only: Gutendex
|
||||||
|
/// writes `Shelley, Mary Wollstonecraft`, Open Library `Mary Shelley`; both give
|
||||||
|
/// `shelley`.
|
||||||
|
pub fn surname(raw: &str) -> String {
|
||||||
|
let raw = raw.trim();
|
||||||
|
let last = match raw.split_once(',') {
|
||||||
|
Some((last, _)) => last,
|
||||||
|
None => raw.rsplit(char::is_whitespace).next().unwrap_or_default(),
|
||||||
|
};
|
||||||
|
last.chars()
|
||||||
|
.filter(|c| c.is_alphanumeric())
|
||||||
|
.collect::<String>()
|
||||||
|
.to_lowercase()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does `name` carry `surname`? Compared with the separators removed, so a
|
||||||
|
/// compound surname (`Saint-Exupéry`) matches however it is spaced.
|
||||||
|
fn author_has_surname(name: &str, surname: &str) -> bool {
|
||||||
|
let flat = name
|
||||||
|
.chars()
|
||||||
|
.filter(|c| c.is_alphanumeric())
|
||||||
|
.collect::<String>()
|
||||||
|
.to_lowercase();
|
||||||
|
!surname.is_empty() && flat.contains(surname)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One document of the title search, with the edition count the choice rests on.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct TitleDoc {
|
||||||
|
#[serde(default)]
|
||||||
|
key: String,
|
||||||
|
#[serde(default)]
|
||||||
|
title: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
author_name: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
edition_count: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide a title binding from a `title=…&author=…` response. See
|
||||||
|
/// [`OpenLibraryClient::title_work`].
|
||||||
|
fn best_title_work(body: &Value, want: &str, surname: &str) -> Option<String> {
|
||||||
|
let docs: Vec<TitleDoc> = body
|
||||||
|
.get("docs")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|d| serde_json::from_value::<TitleDoc>(d.clone()).ok())
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut best: Option<(u64, String)> = None;
|
||||||
|
let mut tie = false;
|
||||||
|
for d in docs {
|
||||||
|
let Some(id) = d.key.strip_prefix("/works/").filter(|id| is_work_id(id)) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if main_title(d.title.as_deref().unwrap_or_default()) != want {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !d.author_name.iter().any(|a| author_has_surname(a, surname)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let n = d.edition_count.unwrap_or(0);
|
||||||
|
let top = best.as_ref().map(|(m, _)| *m);
|
||||||
|
if top.is_none_or(|m| n > m) {
|
||||||
|
best = Some((n, id.to_string()));
|
||||||
|
tie = false;
|
||||||
|
} else if top == Some(n) {
|
||||||
|
tie = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tie {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
best.map(|(_, id)| id)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
||||||
@@ -698,6 +912,70 @@ fn clean_description(raw: &str) -> Option<String> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_main_title_drops_the_subtitle_case_and_punctuation() {
|
||||||
|
assert_eq!(main_title("Frankenstein; Or, The Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein; or, The Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein, or the Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein (Oxford Playscripts)"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Alice's Adventures in Wonderland"), "alice s adventures in wonderland");
|
||||||
|
assert_eq!(main_title(""), "");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_surname_reads_both_name_orders() {
|
||||||
|
assert_eq!(surname("Shelley, Mary Wollstonecraft"), "shelley");
|
||||||
|
assert_eq!(surname("Mary Shelley"), "shelley");
|
||||||
|
assert_eq!(surname(" Austen, Jane "), "austen");
|
||||||
|
assert_eq!(surname("Saint-Exupéry, Antoine de"), "saintexupéry");
|
||||||
|
assert!(author_has_surname("Antoine de Saint-Exupéry", "saintexupéry"));
|
||||||
|
assert!(!author_has_surname("Jason Cobley", "shelley"));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn title_doc(key: &str, title: &str, author: &str, editions: u64) -> Value {
|
||||||
|
serde_json::json!({ "key": key, "title": title, "author_name": [author], "edition_count": editions })
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_title_binding_takes_the_most_published_work_of_that_title_and_author() {
|
||||||
|
let body = serde_json::json!({ "docs": [
|
||||||
|
title_doc("/works/OL26860869W", "Frankenstein", "Mary Shelley", 84),
|
||||||
|
title_doc("/works/OL450063W", "Frankenstein; or, The Modern Prometheus", "Mary Shelley", 2187),
|
||||||
|
title_doc("/works/OL25595002W", "Mary Shelley's Frankenstein", "Mary Shelley", 9000),
|
||||||
|
title_doc("/works/OL15443537W", "Frankenstein", "Dean Koontz", 5000),
|
||||||
|
]});
|
||||||
|
// The 9000-edition work has another title, the 5000 one another author.
|
||||||
|
assert_eq!(best_title_work(&body, "frankenstein", "shelley").as_deref(), Some("OL450063W"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_tie_or_no_match_binds_nothing() {
|
||||||
|
let tie = serde_json::json!({ "docs": [
|
||||||
|
title_doc("/works/OL1W", "Frankenstein", "Mary Shelley", 10),
|
||||||
|
title_doc("/works/OL2W", "Frankenstein", "Mary Shelley", 10),
|
||||||
|
]});
|
||||||
|
assert_eq!(best_title_work(&tie, "frankenstein", "shelley"), None);
|
||||||
|
let other = serde_json::json!({ "docs": [title_doc("/works/OL3W", "Dracula", "Bram Stoker", 99)] });
|
||||||
|
assert_eq!(best_title_work(&other, "frankenstein", "shelley"), None);
|
||||||
|
let not_a_work = serde_json::json!({ "docs": [title_doc("/books/OL4M", "Frankenstein", "Mary Shelley", 99)] });
|
||||||
|
assert_eq!(best_title_work(¬_a_work, "frankenstein", "shelley"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_etext_is_one_several_or_unlisted() {
|
||||||
|
let one = serde_json::json!({ "numFound": 1, "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }] });
|
||||||
|
assert_eq!(etext_of(&one, 1342), Etext::One("OL66554W".into()));
|
||||||
|
let several = serde_json::json!({ "numFound": 2, "docs": [
|
||||||
|
{ "key": "/works/OL450063W", "id_project_gutenberg": ["84"] },
|
||||||
|
{ "key": "/works/OL18388W", "id_project_gutenberg": ["84"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(etext_of(&several, 84), Etext::Several);
|
||||||
|
assert_eq!(etext_of(&serde_json::json!({ "numFound": 0, "docs": [] }), 84), Etext::Unlisted);
|
||||||
|
for e in [Etext::One("OL1W".into()), Etext::Several, Etext::Unlisted] {
|
||||||
|
assert_eq!(Etext::from_cached(&e.to_cached()), Some(e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn work_ids_are_works_only() {
|
fn work_ids_are_works_only() {
|
||||||
assert!(is_work_id("OL66554W"));
|
assert!(is_work_id("OL66554W"));
|
||||||
|
|||||||
Reference in New Issue
Block a user