Compare commits
7
Commits
8bf94cb186
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e6a9ab9e45 | ||
|
|
02bff131ff | ||
|
|
560effe226 | ||
|
|
e14797e57b | ||
|
|
efba2f2825 | ||
|
|
d177e7b710 | ||
|
|
6bfe36edb5 |
@@ -2,10 +2,10 @@
|
|||||||
# folder exists, and GitHub never reads it.
|
# folder exists, and GitHub never reads it.
|
||||||
#
|
#
|
||||||
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
||||||
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
# push ipcr-push-85-17-246-67.ip.nsl.sh/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
||||||
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
||||||
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
||||||
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
# The push goes to ipcr-push-85-17-246-67.ip.nsl.sh, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
||||||
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
||||||
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
||||||
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
||||||
@@ -27,7 +27,7 @@ on:
|
|||||||
tags: ['v*']
|
tags: ['v*']
|
||||||
|
|
||||||
env:
|
env:
|
||||||
REGISTRY: ipcr-push-85-17-246-67.nip.io
|
REGISTRY: ipcr-push-85-17-246-67.ip.nsl.sh
|
||||||
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
@@ -71,6 +71,9 @@ jobs:
|
|||||||
tags: |
|
tags: |
|
||||||
type=ref,event=branch
|
type=ref,event=branch
|
||||||
type=semver,pattern={{version}}
|
type=semver,pattern={{version}}
|
||||||
|
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||||
|
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||||
|
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
type=semver,pattern={{major}}
|
type=semver,pattern={{major}}
|
||||||
type=sha,format=short
|
type=sha,format=short
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# Runs the feeder's tests on every push. The repo had only the image workflow,
|
||||||
|
# so nothing checked a change before it shipped.
|
||||||
|
|
||||||
|
name: Test
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: ["**"]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
rust:
|
||||||
|
name: cargo test
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# The job ran out of memory linking the test binaries (ld killed with
|
||||||
|
# signal 9, run 705 twice): no debug info, and at most two links at once.
|
||||||
|
# What is tested does not change.
|
||||||
|
env:
|
||||||
|
CARGO_PROFILE_DEV_DEBUG: "0"
|
||||||
|
CARGO_BUILD_JOBS: "2"
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
# rust-toolchain.toml pins the channel (with clippy); rustup installs it on
|
||||||
|
# first use, so the action only has to provide rustup itself.
|
||||||
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
with:
|
||||||
|
components: clippy
|
||||||
|
- uses: Swatinem/rust-cache@v2
|
||||||
|
- name: cargo test
|
||||||
|
run: cargo test --locked
|
||||||
|
- name: cargo clippy
|
||||||
|
run: cargo clippy --all-targets -- -D warnings
|
||||||
@@ -43,6 +43,9 @@ jobs:
|
|||||||
tags: |
|
tags: |
|
||||||
type=ref,event=branch
|
type=ref,event=branch
|
||||||
type=semver,pattern={{version}}
|
type=semver,pattern={{version}}
|
||||||
|
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||||
|
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||||
|
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
type=semver,pattern={{major}}
|
type=semver,pattern={{major}}
|
||||||
type=sha,format=short
|
type=sha,format=short
|
||||||
|
|||||||
Generated
+1
-1
@@ -792,7 +792,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "meta-feeder-sdk"
|
name = "meta-feeder-sdk"
|
||||||
version = "1.7.0"
|
version = "1.7.0"
|
||||||
source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
source = "git+https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ path = "src/main.rs"
|
|||||||
# vendored copy this used to point at had already drifted behind the shared SDK
|
# vendored copy this used to point at had already drifted behind the shared SDK
|
||||||
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
||||||
# way; see meta-feeder-anilist's manifest.
|
# way; see meta-feeder-anilist's manifest.
|
||||||
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
|
meta-feeder-sdk = { git = "https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk", tag = "v1.7.0" }
|
||||||
|
|
||||||
tokio = { version = "1.40", features = ["full"] }
|
tokio = { version = "1.40", features = ["full"] }
|
||||||
axum = { version = "0.7", features = ["json"] }
|
axum = { version = "0.7", features = ["json"] }
|
||||||
|
|||||||
@@ -32,7 +32,37 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use crate::openlibrary_client::{self, OpenLibraryClient};
|
use crate::openlibrary::first_filter;
|
||||||
|
use crate::openlibrary_client::{self, Etext, OpenLibraryClient};
|
||||||
|
|
||||||
|
/// An edition's Open Library work, and how it was found.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct Binding {
|
||||||
|
work: String,
|
||||||
|
/// Bound by title and author ([`OpenLibraryClient::title_work`]) rather than
|
||||||
|
/// by an identifier Open Library records: graded Matched, not Anchored.
|
||||||
|
by_title: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One etext's binding: the identifier Open Library records, else — only for an
|
||||||
|
/// etext it lists under no work — the title match. Shared by the search path and
|
||||||
|
/// the redeem path so the two cannot bind the same book differently.
|
||||||
|
async fn bind(
|
||||||
|
client: &OpenLibraryClient,
|
||||||
|
id: u64,
|
||||||
|
title: &str,
|
||||||
|
author: &str,
|
||||||
|
deadline: std::time::Duration,
|
||||||
|
) -> Result<Option<Binding>, GatewayError> {
|
||||||
|
Ok(match client.gutenberg_etext(id, deadline).await? {
|
||||||
|
Etext::One(work) => Some(Binding { work, by_title: false }),
|
||||||
|
Etext::Several => None,
|
||||||
|
Etext::Unlisted => client
|
||||||
|
.title_work(title, author, deadline)
|
||||||
|
.await?
|
||||||
|
.map(|work| Binding { work, by_title: true }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
||||||
/// for tests + private mirrors.
|
/// for tests + private mirrors.
|
||||||
@@ -390,31 +420,60 @@ impl GutenbergPlugin {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The Gutendex search a work page's query stands for.
|
||||||
|
///
|
||||||
|
/// meta-read's book page asks for `openlibraryid:OL…W` with no free text.
|
||||||
|
/// That used to fall through to the catalogue browse — the 32 most
|
||||||
|
/// downloaded books — and the gateway kept those bound to the work, so a
|
||||||
|
/// book outside that list never showed an edition on its own page. Searching
|
||||||
|
/// for the work's main title and its first author's surname finds it
|
||||||
|
/// wherever it ranks; the gateway still keeps only the records bound to the
|
||||||
|
/// work. `None` (no work named, Open Library off or failing) keeps the browse.
|
||||||
|
async fn work_search_text(&self, query: &GatewayQuery) -> Option<String> {
|
||||||
|
let work = first_filter(query, "openlibraryid")?.to_ascii_uppercase();
|
||||||
|
if !openlibrary_client::is_work_id(&work) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let client = self.openlibrary.as_ref()?;
|
||||||
|
let lookup = client.doc_by_work_id(&work, openlibrary_client::BINDING_DEADLINE);
|
||||||
|
let doc = tokio::time::timeout(BINDING_TIMEOUT, lookup).await.ok()?.ok()??;
|
||||||
|
let title = openlibrary_client::main_title(doc.title.as_deref()?);
|
||||||
|
if title.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let author = doc.author_name.first().map(|a| openlibrary_client::surname(a)).unwrap_or_default();
|
||||||
|
Some(format!("{title} {author}").trim().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// Each result's Open Library work, through the Project Gutenberg id Open
|
/// Each result's Open Library work, through the Project Gutenberg id Open
|
||||||
/// Library records (reading-model §2.4). Aligned with the input slice;
|
/// Library records (reading-model §2.4), or, for an etext Open Library lists
|
||||||
/// `None` means no binding — none recorded, ambiguous, not configured, over
|
/// under no work, by title and author (`by_title`, graded Matched). Aligned
|
||||||
/// budget, or the lookup failed.
|
/// with the input slice; `None` means no binding — ambiguous, no title match,
|
||||||
|
/// not configured, over budget, or the lookup failed.
|
||||||
///
|
///
|
||||||
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
||||||
/// edition that is already readable. Every failure is left uncached, so the
|
/// edition that is already readable. Every failure is left uncached, so the
|
||||||
/// next query that surfaces the book tries again; the budget deadline is
|
/// next query that surfaces the book tries again; the budget deadline is
|
||||||
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
||||||
/// first handful of books and the cache fills the rest over later queries.
|
/// first handful of books and the cache fills the rest over later queries.
|
||||||
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<String>> {
|
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<Binding>> {
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
const BINDING_CONCURRENCY: usize = 4;
|
const BINDING_CONCURRENCY: usize = 4;
|
||||||
let Some(client) = self.openlibrary.as_ref() else {
|
let Some(client) = self.openlibrary.as_ref() else {
|
||||||
return vec![None; books.len()];
|
return vec![None; books.len()];
|
||||||
};
|
};
|
||||||
// Owned ids, not `books.iter().map(..)`: a borrowing closure inside the
|
// Owned values, not `books.iter().map(..)`: a borrowing closure inside the
|
||||||
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
||||||
// check ("FnOnce is not general enough").
|
// check ("FnOnce is not general enough").
|
||||||
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
let wanted: Vec<(u64, String, String)> = books
|
||||||
stream::iter(ids)
|
.iter()
|
||||||
.map(|id| async move {
|
.map(|b| (b.id, b.title.clone(), b.authors.first().map(|a| a.name.clone()).unwrap_or_default()))
|
||||||
let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE);
|
.collect();
|
||||||
|
stream::iter(wanted)
|
||||||
|
.map(|(id, title, author)| async move {
|
||||||
|
let lookup = bind(client, id, &title, &author, openlibrary_client::BINDING_DEADLINE);
|
||||||
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
||||||
Ok(Ok(work)) => work,
|
Ok(Ok(binding)) => binding,
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
debug!(
|
debug!(
|
||||||
target: "meta-share::gateway",
|
target: "meta-share::gateway",
|
||||||
@@ -507,7 +566,16 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// everything, already ordered by download count (identical to
|
// everything, already ordered by download count (identical to
|
||||||
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
||||||
// with whatever happens to contain punctuation.
|
// with whatever happens to contain punctuation.
|
||||||
let q = query.free_text.trim();
|
// A work page's query names its Open Library work and no text: search
|
||||||
|
// for that work's title and author instead of browsing. See
|
||||||
|
// `work_search_text`.
|
||||||
|
let free = query.free_text.trim();
|
||||||
|
let work_text = if free.is_empty() {
|
||||||
|
self.work_search_text(query).await
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let q = work_text.as_deref().unwrap_or(free);
|
||||||
let params: Vec<(&str, &str)> = if q.is_empty() {
|
let params: Vec<(&str, &str)> = if q.is_empty() {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
} else {
|
} else {
|
||||||
@@ -568,9 +636,9 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// Not user-facing, so the binding lookup may queue for the full search
|
// Not user-facing, so the binding lookup may queue for the full search
|
||||||
// deadline. A failure still leaves the record unbound, never fails the
|
// deadline. A failure still leaves the record unbound, never fails the
|
||||||
// compute.
|
// compute.
|
||||||
|
let author = book.authors.first().map(|a| a.name.clone()).unwrap_or_default();
|
||||||
let work = match self.openlibrary.as_ref() {
|
let work = match self.openlibrary.as_ref() {
|
||||||
Some(ol) => ol
|
Some(ol) => bind(ol, book.id, &book.title, &author, openlibrary_client::SEARCH_DEADLINE)
|
||||||
.gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE)
|
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -644,13 +712,13 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
|
|
||||||
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
||||||
///
|
///
|
||||||
/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]:
|
/// `openlibrary_work` is the binding from [`bind`]: the one work Open Library
|
||||||
/// `Some` only when Open Library records exactly one work for this etext.
|
/// records for this etext, or, when it records none, the title match.
|
||||||
fn into_discovery_record(
|
fn into_discovery_record(
|
||||||
book: GutendexBook,
|
book: GutendexBook,
|
||||||
cover_cid: Option<String>,
|
cover_cid: Option<String>,
|
||||||
bibrec: BTreeMap<String, String>,
|
bibrec: BTreeMap<String, String>,
|
||||||
openlibrary_work: Option<String>,
|
openlibrary_work: Option<Binding>,
|
||||||
) -> DiscoveryRecord {
|
) -> DiscoveryRecord {
|
||||||
let GutendexBook {
|
let GutendexBook {
|
||||||
id,
|
id,
|
||||||
@@ -803,16 +871,25 @@ fn into_discovery_record(
|
|||||||
let _ = download_count;
|
let _ = download_count;
|
||||||
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
||||||
|
|
||||||
// The work binding and its trust claim (reading-model §2.4). Written only on
|
// The work binding and its trust claim (reading-model §2.4). On an
|
||||||
// an identifier link Open Library itself records, so the claim is the plain
|
// identifier link Open Library itself records, the claim is the plain
|
||||||
// Anchored form: `anchored` + `anchorSource`, and deliberately NO
|
// Anchored form: `anchored` + `anchorSource`, and NO `anchorMethod` — that
|
||||||
// `anchorMethod` — that marks a title match, which this never is. The
|
// marks a title match. The
|
||||||
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
||||||
// strips the transient keys before storing.
|
// strips the transient keys before storing.
|
||||||
if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) {
|
//
|
||||||
fields.insert("openlibraryid".into(), work);
|
// A title binding (an etext Open Library lists under no work) carries the
|
||||||
|
// same claim plus `anchorMethod=title`: the gateway then records
|
||||||
|
// `matchedBy/openlibrary:OL…W` instead of `anchoredBy/`, and meta-read grades
|
||||||
|
// the edition Matched — listed on its work's page, ranked after any
|
||||||
|
// identifier-bound edition.
|
||||||
|
if let Some(binding) = openlibrary_work.filter(|b| openlibrary_client::is_work_id(&b.work)) {
|
||||||
|
fields.insert("openlibraryid".into(), binding.work);
|
||||||
fields.insert("anchored".into(), "true".into());
|
fields.insert("anchored".into(), "true".into());
|
||||||
fields.insert("anchorSource".into(), "openlibrary".into());
|
fields.insert("anchorSource".into(), "openlibrary".into());
|
||||||
|
if binding.by_title {
|
||||||
|
fields.insert("anchorMethod".into(), "title".into());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (k, v) in bibrec {
|
for (k, v) in bibrec {
|
||||||
@@ -1678,6 +1755,98 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open Library's answer to `title=…&author=…` for Frankenstein: several
|
||||||
|
/// works by Mary Shelley share the title; the most published is the novel.
|
||||||
|
fn frankenstein_by_title() -> Mock {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("title", "frankenstein"))
|
||||||
|
.and(query_param("author", "shelley"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 3, "docs": [
|
||||||
|
{ "key": "/works/OL26860869W", "title": "Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 84 },
|
||||||
|
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"], "edition_count": 2187 },
|
||||||
|
{ "key": "/works/OL25595002W", "title": "Mary Shelley's Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 90 }
|
||||||
|
]})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ⚠ Open Library lists etext 84 under no work, so the identifier rule alone
|
||||||
|
/// left the most read novel on Gutenberg unbound and its book page empty.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_etext_no_work_lists_is_bound_by_title_and_graded_matched() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(
|
||||||
|
1342,
|
||||||
|
serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }
|
||||||
|
]}),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
frankenstein_by_title().expect(1).mount(&server).await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("handle_query");
|
||||||
|
|
||||||
|
let pride = &records[0].fields;
|
||||||
|
assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W"));
|
||||||
|
assert!(!pride.contains_key("anchorMethod"), "an identifier binding stays Anchored");
|
||||||
|
|
||||||
|
let frankenstein = &records[1].fields;
|
||||||
|
assert_eq!(frankenstein.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
||||||
|
assert_eq!(frankenstein.get("anchored").map(String::as_str), Some("true"));
|
||||||
|
assert_eq!(frankenstein.get("anchorSource").map(String::as_str), Some("openlibrary"));
|
||||||
|
assert_eq!(frankenstein.get("anchorMethod").map(String::as_str), Some("title"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ⚠ A work page asks for its work and no text. That used to browse the 32
|
||||||
|
/// most downloaded books, so a book outside them never reached its own page.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_work_page_query_searches_for_the_works_title_and_author() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "key:/works/OL450063W"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"] }
|
||||||
|
]})))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.and(query_param("search", "frankenstein shelley"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(1342, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
frankenstein_by_title().mount(&server).await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let mut query = GatewayQuery::from_free_text("");
|
||||||
|
query.filters.insert("openlibraryid".into(), vec!["OL450063W".into()]);
|
||||||
|
query.filters.insert("contentKind".into(), vec!["book".into()]);
|
||||||
|
let records = plugin.handle_query(&query, 50).await.expect("handle_query");
|
||||||
|
let frankenstein = records.iter().find(|r| r.record_id == "84").expect("etext 84 is found");
|
||||||
|
assert_eq!(frankenstein.fields.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
||||||
|
assert_eq!(frankenstein.fields.get("anchorMethod").map(String::as_str), Some("title"));
|
||||||
|
}
|
||||||
|
|
||||||
/// ⚠ A binding failure never costs the record.
|
/// ⚠ A binding failure never costs the record.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ fn env_usize(key: &str, default: usize) -> usize {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// First non-empty value for `key`.
|
/// First non-empty value for `key`.
|
||||||
fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
pub(crate) fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
||||||
q.filters
|
q.filters
|
||||||
.get(key)
|
.get(key)
|
||||||
.and_then(|v| v.first())
|
.and_then(|v| v.first())
|
||||||
|
|||||||
@@ -614,6 +614,248 @@ impl OpenLibraryClient {
|
|||||||
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
Ok(work)
|
Ok(work)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// What Open Library records about an etext: the one work that lists it,
|
||||||
|
/// several works, or none.
|
||||||
|
///
|
||||||
|
/// Same lookup as [`Self::gutenberg_work`], but it keeps the difference
|
||||||
|
/// between "several" and "none", which that one folds into `None`: only an
|
||||||
|
/// etext no work lists may be bound by title ([`Self::title_work`]). Several
|
||||||
|
/// works listing one etext is an unmerged duplicate, and guessing between
|
||||||
|
/// them is exactly what the identifier rule refuses.
|
||||||
|
pub async fn gutenberg_etext(
|
||||||
|
&self,
|
||||||
|
gutenberg_id: u64,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Etext, GatewayError> {
|
||||||
|
let key = format!("openlibrary:etext:{gutenberg_id}");
|
||||||
|
if let Some(c) = self.cache_get(&key) {
|
||||||
|
if let Some(etext) = Etext::from_cached(&c.body) {
|
||||||
|
let ttl = if matches!(etext, Etext::One(_)) {
|
||||||
|
BINDING_HIT_TTL_SECS
|
||||||
|
} else {
|
||||||
|
BINDING_MISS_TTL_SECS
|
||||||
|
};
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl {
|
||||||
|
return Ok(etext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self
|
||||||
|
.get(
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("id_project_gutenberg:{gutenberg_id}")),
|
||||||
|
("fields", "key,id_project_gutenberg".into()),
|
||||||
|
("limit", "2".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let etext = etext_of(&body, gutenberg_id);
|
||||||
|
self.cache_put(&key, &etext.to_cached());
|
||||||
|
Ok(etext)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The work a book is bound to by **title and author**, for an etext Open
|
||||||
|
/// Library lists under no work.
|
||||||
|
///
|
||||||
|
/// Open Library's `id_project_gutenberg` links are patchy even for the most
|
||||||
|
/// read books — none of the editions of *Frankenstein; or, The Modern
|
||||||
|
/// Prometheus* (OL450063W) lists etext 84 — so the identifier rule alone left
|
||||||
|
/// them unbound, and their book pages, which ask for `openlibraryid:OL…W`,
|
||||||
|
/// empty. This is the weaker proof the trust model already has a grade for:
|
||||||
|
/// the caller stamps `anchorMethod=title`, the gateway records
|
||||||
|
/// `matchedBy/openlibrary:OL…W`, and meta-read ranks such an edition after an
|
||||||
|
/// identifier-bound one.
|
||||||
|
///
|
||||||
|
/// Same main title ([`main_title`]), an author with the same surname, and,
|
||||||
|
/// among those, the work with the most editions — Open Library holds many
|
||||||
|
/// small works titled *Frankenstein* by Mary Shelley (adaptations, annotated
|
||||||
|
/// and school editions), and the canonical one is the one most published.
|
||||||
|
/// `None` when nothing matches or two works tie.
|
||||||
|
pub async fn title_work(
|
||||||
|
&self,
|
||||||
|
title: &str,
|
||||||
|
author: &str,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Option<String>, GatewayError> {
|
||||||
|
let (want, surname) = (main_title(title), surname(author));
|
||||||
|
if want.is_empty() || surname.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let key = format!("openlibrary:titlework:{want}:{surname}");
|
||||||
|
if let Some(c) = self.cache_get(&key) {
|
||||||
|
let work = c.body.get("work").and_then(Value::as_str).map(str::to_string);
|
||||||
|
let ttl = if work.is_some() {
|
||||||
|
BINDING_HIT_TTL_SECS
|
||||||
|
} else {
|
||||||
|
BINDING_MISS_TTL_SECS
|
||||||
|
};
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl {
|
||||||
|
return Ok(work);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self
|
||||||
|
.get(
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("title", want.clone()),
|
||||||
|
("author", surname.clone()),
|
||||||
|
("fields", "key,title,author_name,edition_count".into()),
|
||||||
|
("limit", "20".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let work = best_title_work(&body, &want, &surname);
|
||||||
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
|
Ok(work)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What Open Library records about one Gutenberg etext. See
|
||||||
|
/// [`OpenLibraryClient::gutenberg_etext`].
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Etext {
|
||||||
|
/// Exactly one work lists it: an identifier binding, graded Anchored.
|
||||||
|
One(String),
|
||||||
|
/// Several works list it: an unmerged duplicate, so no binding at all.
|
||||||
|
Several,
|
||||||
|
/// No work lists it: the one case a title binding may fill.
|
||||||
|
Unlisted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Etext {
|
||||||
|
fn to_cached(&self) -> Value {
|
||||||
|
match self {
|
||||||
|
Etext::One(work) => serde_json::json!({ "etext": "one", "work": work }),
|
||||||
|
Etext::Several => serde_json::json!({ "etext": "several" }),
|
||||||
|
Etext::Unlisted => serde_json::json!({ "etext": "unlisted" }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_cached(v: &Value) -> Option<Self> {
|
||||||
|
match v.get("etext")?.as_str()? {
|
||||||
|
"one" => Some(Etext::One(v.get("work")?.as_str()?.to_string())),
|
||||||
|
"several" => Some(Etext::Several),
|
||||||
|
"unlisted" => Some(Etext::Unlisted),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn etext_of(body: &Value, gutenberg_id: u64) -> Etext {
|
||||||
|
if let Some(work) = single_gutenberg_work(body, gutenberg_id) {
|
||||||
|
return Etext::One(work);
|
||||||
|
}
|
||||||
|
let docs = parse_docs(body.get("docs"));
|
||||||
|
let total = body
|
||||||
|
.get("numFound")
|
||||||
|
.or_else(|| body.get("num_found"))
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(docs.len() as u64);
|
||||||
|
if total == 0 && docs.is_empty() {
|
||||||
|
Etext::Unlisted
|
||||||
|
} else {
|
||||||
|
Etext::Several
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The part of a title two catalogues agree on: before any subtitle (`;`, `:`,
|
||||||
|
/// a bracket, or `, or`), lower-cased, letters and digits only. Gutenberg's
|
||||||
|
/// *Frankenstein; Or, The Modern Prometheus* and Open Library's *Frankenstein;
|
||||||
|
/// or, The Modern Prometheus* are both `frankenstein`.
|
||||||
|
pub fn main_title(raw: &str) -> String {
|
||||||
|
let lower = raw.to_lowercase();
|
||||||
|
let head = lower
|
||||||
|
.split([';', ':', '(', '['])
|
||||||
|
.next()
|
||||||
|
.unwrap_or_default();
|
||||||
|
let head = head.find(", or").map_or(head, |i| &head[..i]);
|
||||||
|
head.split(|c: char| !c.is_alphanumeric())
|
||||||
|
.filter(|w| !w.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The surname a name ends on, lower-cased, letters and digits only: Gutendex
|
||||||
|
/// writes `Shelley, Mary Wollstonecraft`, Open Library `Mary Shelley`; both give
|
||||||
|
/// `shelley`.
|
||||||
|
pub fn surname(raw: &str) -> String {
|
||||||
|
let raw = raw.trim();
|
||||||
|
let last = match raw.split_once(',') {
|
||||||
|
Some((last, _)) => last,
|
||||||
|
None => raw.rsplit(char::is_whitespace).next().unwrap_or_default(),
|
||||||
|
};
|
||||||
|
last.chars()
|
||||||
|
.filter(|c| c.is_alphanumeric())
|
||||||
|
.collect::<String>()
|
||||||
|
.to_lowercase()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does `name` carry `surname`? Compared with the separators removed, so a
|
||||||
|
/// compound surname (`Saint-Exupéry`) matches however it is spaced.
|
||||||
|
fn author_has_surname(name: &str, surname: &str) -> bool {
|
||||||
|
let flat = name
|
||||||
|
.chars()
|
||||||
|
.filter(|c| c.is_alphanumeric())
|
||||||
|
.collect::<String>()
|
||||||
|
.to_lowercase();
|
||||||
|
!surname.is_empty() && flat.contains(surname)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One document of the title search, with the edition count the choice rests on.
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
struct TitleDoc {
|
||||||
|
#[serde(default)]
|
||||||
|
key: String,
|
||||||
|
#[serde(default)]
|
||||||
|
title: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
author_name: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
edition_count: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide a title binding from a `title=…&author=…` response. See
|
||||||
|
/// [`OpenLibraryClient::title_work`].
|
||||||
|
fn best_title_work(body: &Value, want: &str, surname: &str) -> Option<String> {
|
||||||
|
let docs: Vec<TitleDoc> = body
|
||||||
|
.get("docs")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|d| serde_json::from_value::<TitleDoc>(d.clone()).ok())
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut best: Option<(u64, String)> = None;
|
||||||
|
let mut tie = false;
|
||||||
|
for d in docs {
|
||||||
|
let Some(id) = d.key.strip_prefix("/works/").filter(|id| is_work_id(id)) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if main_title(d.title.as_deref().unwrap_or_default()) != want {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !d.author_name.iter().any(|a| author_has_surname(a, surname)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let n = d.edition_count.unwrap_or(0);
|
||||||
|
let top = best.as_ref().map(|(m, _)| *m);
|
||||||
|
if top.is_none_or(|m| n > m) {
|
||||||
|
best = Some((n, id.to_string()));
|
||||||
|
tie = false;
|
||||||
|
} else if top == Some(n) {
|
||||||
|
tie = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tie {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
best.map(|(_, id)| id)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
||||||
@@ -670,6 +912,70 @@ fn clean_description(raw: &str) -> Option<String> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_main_title_drops_the_subtitle_case_and_punctuation() {
|
||||||
|
assert_eq!(main_title("Frankenstein; Or, The Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein; or, The Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein, or the Modern Prometheus"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Frankenstein (Oxford Playscripts)"), "frankenstein");
|
||||||
|
assert_eq!(main_title("Alice's Adventures in Wonderland"), "alice s adventures in wonderland");
|
||||||
|
assert_eq!(main_title(""), "");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_surname_reads_both_name_orders() {
|
||||||
|
assert_eq!(surname("Shelley, Mary Wollstonecraft"), "shelley");
|
||||||
|
assert_eq!(surname("Mary Shelley"), "shelley");
|
||||||
|
assert_eq!(surname(" Austen, Jane "), "austen");
|
||||||
|
assert_eq!(surname("Saint-Exupéry, Antoine de"), "saintexupéry");
|
||||||
|
assert!(author_has_surname("Antoine de Saint-Exupéry", "saintexupéry"));
|
||||||
|
assert!(!author_has_surname("Jason Cobley", "shelley"));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn title_doc(key: &str, title: &str, author: &str, editions: u64) -> Value {
|
||||||
|
serde_json::json!({ "key": key, "title": title, "author_name": [author], "edition_count": editions })
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_title_binding_takes_the_most_published_work_of_that_title_and_author() {
|
||||||
|
let body = serde_json::json!({ "docs": [
|
||||||
|
title_doc("/works/OL26860869W", "Frankenstein", "Mary Shelley", 84),
|
||||||
|
title_doc("/works/OL450063W", "Frankenstein; or, The Modern Prometheus", "Mary Shelley", 2187),
|
||||||
|
title_doc("/works/OL25595002W", "Mary Shelley's Frankenstein", "Mary Shelley", 9000),
|
||||||
|
title_doc("/works/OL15443537W", "Frankenstein", "Dean Koontz", 5000),
|
||||||
|
]});
|
||||||
|
// The 9000-edition work has another title, the 5000 one another author.
|
||||||
|
assert_eq!(best_title_work(&body, "frankenstein", "shelley").as_deref(), Some("OL450063W"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_tie_or_no_match_binds_nothing() {
|
||||||
|
let tie = serde_json::json!({ "docs": [
|
||||||
|
title_doc("/works/OL1W", "Frankenstein", "Mary Shelley", 10),
|
||||||
|
title_doc("/works/OL2W", "Frankenstein", "Mary Shelley", 10),
|
||||||
|
]});
|
||||||
|
assert_eq!(best_title_work(&tie, "frankenstein", "shelley"), None);
|
||||||
|
let other = serde_json::json!({ "docs": [title_doc("/works/OL3W", "Dracula", "Bram Stoker", 99)] });
|
||||||
|
assert_eq!(best_title_work(&other, "frankenstein", "shelley"), None);
|
||||||
|
let not_a_work = serde_json::json!({ "docs": [title_doc("/books/OL4M", "Frankenstein", "Mary Shelley", 99)] });
|
||||||
|
assert_eq!(best_title_work(¬_a_work, "frankenstein", "shelley"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_etext_is_one_several_or_unlisted() {
|
||||||
|
let one = serde_json::json!({ "numFound": 1, "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }] });
|
||||||
|
assert_eq!(etext_of(&one, 1342), Etext::One("OL66554W".into()));
|
||||||
|
let several = serde_json::json!({ "numFound": 2, "docs": [
|
||||||
|
{ "key": "/works/OL450063W", "id_project_gutenberg": ["84"] },
|
||||||
|
{ "key": "/works/OL18388W", "id_project_gutenberg": ["84"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(etext_of(&several, 84), Etext::Several);
|
||||||
|
assert_eq!(etext_of(&serde_json::json!({ "numFound": 0, "docs": [] }), 84), Etext::Unlisted);
|
||||||
|
for e in [Etext::One("OL1W".into()), Etext::Several, Etext::Unlisted] {
|
||||||
|
assert_eq!(Etext::from_cached(&e.to_cached()), Some(e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn work_ids_are_works_only() {
|
fn work_ids_are_works_only() {
|
||||||
assert!(is_work_id("OL66554W"));
|
assert!(is_work_id("OL66554W"));
|
||||||
|
|||||||
Reference in New Issue
Block a user