Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3564f170b4 |
@@ -1,31 +0,0 @@
|
|||||||
# Runs the feeder's tests on every push. The repo had only the image workflow,
|
|
||||||
# so nothing checked a change before it shipped.
|
|
||||||
|
|
||||||
name: Test
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: ["**"]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
rust:
|
|
||||||
name: cargo test
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# The job ran out of memory linking the test binaries (ld killed with
|
|
||||||
# signal 9, run 705 twice): no debug info, and at most two links at once.
|
|
||||||
# What is tested does not change.
|
|
||||||
env:
|
|
||||||
CARGO_PROFILE_DEV_DEBUG: "0"
|
|
||||||
CARGO_BUILD_JOBS: "2"
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
# rust-toolchain.toml pins the channel (with clippy); rustup installs it on
|
|
||||||
# first use, so the action only has to provide rustup itself.
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
|
||||||
with:
|
|
||||||
components: clippy
|
|
||||||
- uses: Swatinem/rust-cache@v2
|
|
||||||
- name: cargo test
|
|
||||||
run: cargo test --locked
|
|
||||||
- name: cargo clippy
|
|
||||||
run: cargo clippy --all-targets -- -D warnings
|
|
||||||
Generated
+1
-1
@@ -792,7 +792,7 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "meta-feeder-sdk"
|
name = "meta-feeder-sdk"
|
||||||
version = "1.7.0"
|
version = "1.7.0"
|
||||||
source = "git+https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ path = "src/main.rs"
|
|||||||
# vendored copy this used to point at had already drifted behind the shared SDK
|
# vendored copy this used to point at had already drifted behind the shared SDK
|
||||||
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
||||||
# way; see meta-feeder-anilist's manifest.
|
# way; see meta-feeder-anilist's manifest.
|
||||||
meta-feeder-sdk = { git = "https://gitea-watch.nsl.sh/metastar/meta-feeder-sdk", tag = "v1.7.0" }
|
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
|
||||||
|
|
||||||
tokio = { version = "1.40", features = ["full"] }
|
tokio = { version = "1.40", features = ["full"] }
|
||||||
axum = { version = "0.7", features = ["json"] }
|
axum = { version = "0.7", features = ["json"] }
|
||||||
|
|||||||
@@ -32,37 +32,7 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
use crate::openlibrary::first_filter;
|
use crate::openlibrary_client::{self, OpenLibraryClient};
|
||||||
use crate::openlibrary_client::{self, Etext, OpenLibraryClient};
|
|
||||||
|
|
||||||
/// An edition's Open Library work, and how it was found.
|
|
||||||
#[derive(Debug, Clone)]
|
|
||||||
struct Binding {
|
|
||||||
work: String,
|
|
||||||
/// Bound by title and author ([`OpenLibraryClient::title_work`]) rather than
|
|
||||||
/// by an identifier Open Library records: graded Matched, not Anchored.
|
|
||||||
by_title: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One etext's binding: the identifier Open Library records, else — only for an
|
|
||||||
/// etext it lists under no work — the title match. Shared by the search path and
|
|
||||||
/// the redeem path so the two cannot bind the same book differently.
|
|
||||||
async fn bind(
|
|
||||||
client: &OpenLibraryClient,
|
|
||||||
id: u64,
|
|
||||||
title: &str,
|
|
||||||
author: &str,
|
|
||||||
deadline: std::time::Duration,
|
|
||||||
) -> Result<Option<Binding>, GatewayError> {
|
|
||||||
Ok(match client.gutenberg_etext(id, deadline).await? {
|
|
||||||
Etext::One(work) => Some(Binding { work, by_title: false }),
|
|
||||||
Etext::Several => None,
|
|
||||||
Etext::Unlisted => client
|
|
||||||
.title_work(title, author, deadline)
|
|
||||||
.await?
|
|
||||||
.map(|work| Binding { work, by_title: true }),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
||||||
/// for tests + private mirrors.
|
/// for tests + private mirrors.
|
||||||
@@ -420,60 +390,31 @@ impl GutenbergPlugin {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The Gutendex search a work page's query stands for.
|
|
||||||
///
|
|
||||||
/// meta-read's book page asks for `openlibraryid:OL…W` with no free text.
|
|
||||||
/// That used to fall through to the catalogue browse — the 32 most
|
|
||||||
/// downloaded books — and the gateway kept those bound to the work, so a
|
|
||||||
/// book outside that list never showed an edition on its own page. Searching
|
|
||||||
/// for the work's main title and its first author's surname finds it
|
|
||||||
/// wherever it ranks; the gateway still keeps only the records bound to the
|
|
||||||
/// work. `None` (no work named, Open Library off or failing) keeps the browse.
|
|
||||||
async fn work_search_text(&self, query: &GatewayQuery) -> Option<String> {
|
|
||||||
let work = first_filter(query, "openlibraryid")?.to_ascii_uppercase();
|
|
||||||
if !openlibrary_client::is_work_id(&work) {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let client = self.openlibrary.as_ref()?;
|
|
||||||
let lookup = client.doc_by_work_id(&work, openlibrary_client::BINDING_DEADLINE);
|
|
||||||
let doc = tokio::time::timeout(BINDING_TIMEOUT, lookup).await.ok()?.ok()??;
|
|
||||||
let title = openlibrary_client::main_title(doc.title.as_deref()?);
|
|
||||||
if title.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let author = doc.author_name.first().map(|a| openlibrary_client::surname(a)).unwrap_or_default();
|
|
||||||
Some(format!("{title} {author}").trim().to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Each result's Open Library work, through the Project Gutenberg id Open
|
/// Each result's Open Library work, through the Project Gutenberg id Open
|
||||||
/// Library records (reading-model §2.4), or, for an etext Open Library lists
|
/// Library records (reading-model §2.4). Aligned with the input slice;
|
||||||
/// under no work, by title and author (`by_title`, graded Matched). Aligned
|
/// `None` means no binding — none recorded, ambiguous, not configured, over
|
||||||
/// with the input slice; `None` means no binding — ambiguous, no title match,
|
/// budget, or the lookup failed.
|
||||||
/// not configured, over budget, or the lookup failed.
|
|
||||||
///
|
///
|
||||||
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
||||||
/// edition that is already readable. Every failure is left uncached, so the
|
/// edition that is already readable. Every failure is left uncached, so the
|
||||||
/// next query that surfaces the book tries again; the budget deadline is
|
/// next query that surfaces the book tries again; the budget deadline is
|
||||||
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
||||||
/// first handful of books and the cache fills the rest over later queries.
|
/// first handful of books and the cache fills the rest over later queries.
|
||||||
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<Binding>> {
|
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<String>> {
|
||||||
use futures::stream::{self, StreamExt};
|
use futures::stream::{self, StreamExt};
|
||||||
const BINDING_CONCURRENCY: usize = 4;
|
const BINDING_CONCURRENCY: usize = 4;
|
||||||
let Some(client) = self.openlibrary.as_ref() else {
|
let Some(client) = self.openlibrary.as_ref() else {
|
||||||
return vec![None; books.len()];
|
return vec![None; books.len()];
|
||||||
};
|
};
|
||||||
// Owned values, not `books.iter().map(..)`: a borrowing closure inside the
|
// Owned ids, not `books.iter().map(..)`: a borrowing closure inside the
|
||||||
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
||||||
// check ("FnOnce is not general enough").
|
// check ("FnOnce is not general enough").
|
||||||
let wanted: Vec<(u64, String, String)> = books
|
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
||||||
.iter()
|
stream::iter(ids)
|
||||||
.map(|b| (b.id, b.title.clone(), b.authors.first().map(|a| a.name.clone()).unwrap_or_default()))
|
.map(|id| async move {
|
||||||
.collect();
|
let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE);
|
||||||
stream::iter(wanted)
|
|
||||||
.map(|(id, title, author)| async move {
|
|
||||||
let lookup = bind(client, id, &title, &author, openlibrary_client::BINDING_DEADLINE);
|
|
||||||
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
||||||
Ok(Ok(binding)) => binding,
|
Ok(Ok(work)) => work,
|
||||||
Ok(Err(e)) => {
|
Ok(Err(e)) => {
|
||||||
debug!(
|
debug!(
|
||||||
target: "meta-share::gateway",
|
target: "meta-share::gateway",
|
||||||
@@ -566,16 +507,7 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// everything, already ordered by download count (identical to
|
// everything, already ordered by download count (identical to
|
||||||
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
// `?sort=popular`), so the row leads with Pride and Prejudice rather than
|
||||||
// with whatever happens to contain punctuation.
|
// with whatever happens to contain punctuation.
|
||||||
// A work page's query names its Open Library work and no text: search
|
let q = query.free_text.trim();
|
||||||
// for that work's title and author instead of browsing. See
|
|
||||||
// `work_search_text`.
|
|
||||||
let free = query.free_text.trim();
|
|
||||||
let work_text = if free.is_empty() {
|
|
||||||
self.work_search_text(query).await
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
let q = work_text.as_deref().unwrap_or(free);
|
|
||||||
let params: Vec<(&str, &str)> = if q.is_empty() {
|
let params: Vec<(&str, &str)> = if q.is_empty() {
|
||||||
Vec::new()
|
Vec::new()
|
||||||
} else {
|
} else {
|
||||||
@@ -636,9 +568,9 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
// Not user-facing, so the binding lookup may queue for the full search
|
// Not user-facing, so the binding lookup may queue for the full search
|
||||||
// deadline. A failure still leaves the record unbound, never fails the
|
// deadline. A failure still leaves the record unbound, never fails the
|
||||||
// compute.
|
// compute.
|
||||||
let author = book.authors.first().map(|a| a.name.clone()).unwrap_or_default();
|
|
||||||
let work = match self.openlibrary.as_ref() {
|
let work = match self.openlibrary.as_ref() {
|
||||||
Some(ol) => bind(ol, book.id, &book.title, &author, openlibrary_client::SEARCH_DEADLINE)
|
Some(ol) => ol
|
||||||
|
.gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| {
|
.unwrap_or_else(|e| {
|
||||||
debug!(
|
debug!(
|
||||||
@@ -712,13 +644,13 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
|
|
||||||
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
||||||
///
|
///
|
||||||
/// `openlibrary_work` is the binding from [`bind`]: the one work Open Library
|
/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]:
|
||||||
/// records for this etext, or, when it records none, the title match.
|
/// `Some` only when Open Library records exactly one work for this etext.
|
||||||
fn into_discovery_record(
|
fn into_discovery_record(
|
||||||
book: GutendexBook,
|
book: GutendexBook,
|
||||||
cover_cid: Option<String>,
|
cover_cid: Option<String>,
|
||||||
bibrec: BTreeMap<String, String>,
|
bibrec: BTreeMap<String, String>,
|
||||||
openlibrary_work: Option<Binding>,
|
openlibrary_work: Option<String>,
|
||||||
) -> DiscoveryRecord {
|
) -> DiscoveryRecord {
|
||||||
let GutendexBook {
|
let GutendexBook {
|
||||||
id,
|
id,
|
||||||
@@ -871,25 +803,16 @@ fn into_discovery_record(
|
|||||||
let _ = download_count;
|
let _ = download_count;
|
||||||
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
||||||
|
|
||||||
// The work binding and its trust claim (reading-model §2.4). On an
|
// The work binding and its trust claim (reading-model §2.4). Written only on
|
||||||
// identifier link Open Library itself records, the claim is the plain
|
// an identifier link Open Library itself records, so the claim is the plain
|
||||||
// Anchored form: `anchored` + `anchorSource`, and NO `anchorMethod` — that
|
// Anchored form: `anchored` + `anchorSource`, and deliberately NO
|
||||||
// marks a title match. The
|
// `anchorMethod` — that marks a title match, which this never is. The
|
||||||
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
||||||
// strips the transient keys before storing.
|
// strips the transient keys before storing.
|
||||||
//
|
if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) {
|
||||||
// A title binding (an etext Open Library lists under no work) carries the
|
fields.insert("openlibraryid".into(), work);
|
||||||
// same claim plus `anchorMethod=title`: the gateway then records
|
|
||||||
// `matchedBy/openlibrary:OL…W` instead of `anchoredBy/`, and meta-read grades
|
|
||||||
// the edition Matched — listed on its work's page, ranked after any
|
|
||||||
// identifier-bound edition.
|
|
||||||
if let Some(binding) = openlibrary_work.filter(|b| openlibrary_client::is_work_id(&b.work)) {
|
|
||||||
fields.insert("openlibraryid".into(), binding.work);
|
|
||||||
fields.insert("anchored".into(), "true".into());
|
fields.insert("anchored".into(), "true".into());
|
||||||
fields.insert("anchorSource".into(), "openlibrary".into());
|
fields.insert("anchorSource".into(), "openlibrary".into());
|
||||||
if binding.by_title {
|
|
||||||
fields.insert("anchorMethod".into(), "title".into());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (k, v) in bibrec {
|
for (k, v) in bibrec {
|
||||||
@@ -1755,98 +1678,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open Library's answer to `title=…&author=…` for Frankenstein: several
|
|
||||||
/// works by Mary Shelley share the title; the most published is the novel.
|
|
||||||
fn frankenstein_by_title() -> Mock {
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/search.json"))
|
|
||||||
.and(query_param("title", "frankenstein"))
|
|
||||||
.and(query_param("author", "shelley"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 3, "docs": [
|
|
||||||
{ "key": "/works/OL26860869W", "title": "Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 84 },
|
|
||||||
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"], "edition_count": 2187 },
|
|
||||||
{ "key": "/works/OL25595002W", "title": "Mary Shelley's Frankenstein", "author_name": ["Mary Shelley"], "edition_count": 90 }
|
|
||||||
]})))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// ⚠ Open Library lists etext 84 under no work, so the identifier rule alone
|
|
||||||
/// left the most read novel on Gutenberg unbound and its book page empty.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn an_etext_no_work_lists_is_bound_by_title_and_graded_matched() {
|
|
||||||
let server = MockServer::start().await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/books/"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
binding(
|
|
||||||
1342,
|
|
||||||
serde_json::json!({ "numFound": 1, "docs": [
|
|
||||||
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }
|
|
||||||
]}),
|
|
||||||
)
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
frankenstein_by_title().expect(1).mount(&server).await;
|
|
||||||
|
|
||||||
let (plugin, _dir) = configured_plugin_against(&server);
|
|
||||||
let records = plugin
|
|
||||||
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
|
||||||
.await
|
|
||||||
.expect("handle_query");
|
|
||||||
|
|
||||||
let pride = &records[0].fields;
|
|
||||||
assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W"));
|
|
||||||
assert!(!pride.contains_key("anchorMethod"), "an identifier binding stays Anchored");
|
|
||||||
|
|
||||||
let frankenstein = &records[1].fields;
|
|
||||||
assert_eq!(frankenstein.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
|
||||||
assert_eq!(frankenstein.get("anchored").map(String::as_str), Some("true"));
|
|
||||||
assert_eq!(frankenstein.get("anchorSource").map(String::as_str), Some("openlibrary"));
|
|
||||||
assert_eq!(frankenstein.get("anchorMethod").map(String::as_str), Some("title"));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// ⚠ A work page asks for its work and no text. That used to browse the 32
|
|
||||||
/// most downloaded books, so a book outside them never reached its own page.
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_work_page_query_searches_for_the_works_title_and_author() {
|
|
||||||
let server = MockServer::start().await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/search.json"))
|
|
||||||
.and(query_param("q", "key:/works/OL450063W"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 1, "docs": [
|
|
||||||
{ "key": "/works/OL450063W", "title": "Frankenstein; or, The Modern Prometheus", "author_name": ["Mary Shelley"] }
|
|
||||||
]})))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/books/"))
|
|
||||||
.and(query_param("search", "frankenstein shelley"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
|
||||||
.expect(1)
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
binding(1342, serde_json::json!({ "numFound": 0, "docs": [] }))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
frankenstein_by_title().mount(&server).await;
|
|
||||||
|
|
||||||
let (plugin, _dir) = configured_plugin_against(&server);
|
|
||||||
let mut query = GatewayQuery::from_free_text("");
|
|
||||||
query.filters.insert("openlibraryid".into(), vec!["OL450063W".into()]);
|
|
||||||
query.filters.insert("contentKind".into(), vec!["book".into()]);
|
|
||||||
let records = plugin.handle_query(&query, 50).await.expect("handle_query");
|
|
||||||
let frankenstein = records.iter().find(|r| r.record_id == "84").expect("etext 84 is found");
|
|
||||||
assert_eq!(frankenstein.fields.get("openlibraryid").map(String::as_str), Some("OL450063W"));
|
|
||||||
assert_eq!(frankenstein.fields.get("anchorMethod").map(String::as_str), Some("title"));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// ⚠ A binding failure never costs the record.
|
/// ⚠ A binding failure never costs the record.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ fn env_usize(key: &str, default: usize) -> usize {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// First non-empty value for `key`.
|
/// First non-empty value for `key`.
|
||||||
pub(crate) fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
fn first_filter(q: &GatewayQuery, key: &str) -> Option<String> {
|
||||||
q.filters
|
q.filters
|
||||||
.get(key)
|
.get(key)
|
||||||
.and_then(|v| v.first())
|
.and_then(|v| v.first())
|
||||||
@@ -109,62 +109,6 @@ fn browse_kind(q: &GatewayQuery) -> Option<Browse> {
|
|||||||
/// The browse markers a card echoes. See [`to_record`].
|
/// The browse markers a card echoes. See [`to_record`].
|
||||||
const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"];
|
const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"];
|
||||||
|
|
||||||
/// One `genres:<slug>` value: the Open Library subject key it narrows to and
|
|
||||||
/// the name a card stores under `genres/`.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
struct Genre {
|
|
||||||
subject: String,
|
|
||||||
name: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The genres a query asks for, in order; empty when it asks for none.
|
|
||||||
///
|
|
||||||
/// The wire carries a slug (`genres:science-fiction`, meta-read `config.rs`);
|
|
||||||
/// Open Library's subject key is the same words joined by `_`
|
|
||||||
/// (`science_fiction`), and the stored name is the words capitalised
|
|
||||||
/// (`Science Fiction`). The gateway's filter folds both sides to alphanumerics,
|
|
||||||
/// so the name matches the slug.
|
|
||||||
///
|
|
||||||
/// `None` when a value is not a plain slug (`[A-Za-z0-9_-]`): it would be
|
|
||||||
/// pasted into an Open Library query, and a row whose genre cannot be read
|
|
||||||
/// answers nothing rather than every genre.
|
|
||||||
fn requested_genres(q: &GatewayQuery) -> Option<Vec<Genre>> {
|
|
||||||
let mut out: Vec<Genre> = Vec::new();
|
|
||||||
for raw in q.filters.get("genres").into_iter().flatten() {
|
|
||||||
let raw = raw.trim();
|
|
||||||
if raw.is_empty() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if !raw.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let words: Vec<String> = raw
|
|
||||||
.split(['-', '_'])
|
|
||||||
.filter(|w| !w.is_empty())
|
|
||||||
.map(str::to_ascii_lowercase)
|
|
||||||
.collect();
|
|
||||||
if words.is_empty() {
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
let subject = words.join("_");
|
|
||||||
if out.iter().any(|g| g.subject == subject) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let name = words
|
|
||||||
.iter()
|
|
||||||
.map(|w| {
|
|
||||||
let mut c = w.chars();
|
|
||||||
c.next()
|
|
||||||
.map(|f| f.to_ascii_uppercase().to_string() + c.as_str())
|
|
||||||
.unwrap_or_default()
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(" ");
|
|
||||||
out.push(Genre { subject, name });
|
|
||||||
}
|
|
||||||
Some(out)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Identity and instalment filters this feeder cannot answer.
|
/// Identity and instalment filters this feeder cannot answer.
|
||||||
///
|
///
|
||||||
/// ⚠ An id-anchored query this upstream does not understand must return
|
/// ⚠ An id-anchored query this upstream does not understand must return
|
||||||
@@ -224,20 +168,34 @@ fn foreign_identity_filter(q: &GatewayQuery) -> Option<&str> {
|
|||||||
/// would stamp e.g. a `licence:pd` query onto a card — and the gateway stores
|
/// would stamp e.g. a `licence:pd` query onto a card — and the gateway stores
|
||||||
/// what passes, permanently. A card that lacks another filter's key does not
|
/// what passes, permanently. A card that lacks another filter's key does not
|
||||||
/// match it, which is the truth.
|
/// match it, which is the truth.
|
||||||
///
|
|
||||||
/// Genres are written from the doc, not echoed: a requested genre is stamped
|
|
||||||
/// only when the doc's `subject_key` lists it, which only a subject-narrowed
|
|
||||||
/// row fetches. The gateway's genre filter fails closed, so without the stamp
|
|
||||||
/// a genre row renders empty.
|
|
||||||
pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option<DiscoveryRecord> {
|
pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Option<DiscoveryRecord> {
|
||||||
|
to_record_with(doc, Some(description), query, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`to_record`] with the quality bar as a choice.
|
||||||
|
///
|
||||||
|
/// `lenient = false` is the bar every search and browse row keeps: a cover **and**
|
||||||
|
/// a description, or no card. `lenient = true` is for a lookup that names one work
|
||||||
|
/// by `openlibraryid` (a work page): the work is already known to be wanted, so a
|
||||||
|
/// title is enough. Its cover may come from the work's ISBNs when the work has none
|
||||||
|
/// of its own, and a missing description is simply left out.
|
||||||
|
pub fn to_record_with(
|
||||||
|
doc: &SearchDoc,
|
||||||
|
description: Option<&str>,
|
||||||
|
query: &GatewayQuery,
|
||||||
|
lenient: bool,
|
||||||
|
) -> Option<DiscoveryRecord> {
|
||||||
let id = doc.work_id()?;
|
let id = doc.work_id()?;
|
||||||
let cid = card_cid(id)?;
|
let cid = card_cid(id)?;
|
||||||
let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?;
|
let title = doc.title.as_deref().map(str::trim).filter(|t| !t.is_empty())?;
|
||||||
let poster_url = doc.cover_url()?;
|
let description = description.map(str::trim).filter(|d| !d.is_empty());
|
||||||
let description = description.trim();
|
let poster_url = if lenient {
|
||||||
if description.is_empty() {
|
doc.cover_url().or_else(|| doc.isbn_cover_url())
|
||||||
return None;
|
} else {
|
||||||
}
|
// Strict: both are required.
|
||||||
|
description?;
|
||||||
|
Some(doc.cover_url()?)
|
||||||
|
};
|
||||||
|
|
||||||
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
||||||
|
|
||||||
@@ -261,7 +219,9 @@ pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Op
|
|||||||
|
|
||||||
// `description/{lang3}`, never a flat `description` — the flat key is indexed
|
// `description/{lang3}`, never a flat `description` — the flat key is indexed
|
||||||
// but never written (METADATA_KEYS.md §14.6).
|
// but never written (METADATA_KEYS.md §14.6).
|
||||||
fields.insert("description/eng".into(), description.to_string());
|
if let Some(d) = description {
|
||||||
|
fields.insert("description/eng".into(), d.to_string());
|
||||||
|
}
|
||||||
|
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
let authors: Vec<&str> = doc
|
let authors: Vec<&str> = doc
|
||||||
@@ -285,7 +245,9 @@ pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Op
|
|||||||
|
|
||||||
// ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS
|
// ⚠ `poster_url`, NOT a pre-hashed `poster`: the gateway's SEEDABLE_FIELDS
|
||||||
// path fetches it, seeds the bytes and writes the content cid back.
|
// path fetches it, seeds the bytes and writes the content cid back.
|
||||||
fields.insert("poster_url".into(), poster_url);
|
if let Some(url) = poster_url {
|
||||||
|
fields.insert("poster_url".into(), url);
|
||||||
|
}
|
||||||
|
|
||||||
fields.insert(
|
fields.insert(
|
||||||
"sourceUrl".into(),
|
"sourceUrl".into(),
|
||||||
@@ -299,12 +261,6 @@ pub fn to_record(doc: &SearchDoc, description: &str, query: &GatewayQuery) -> Op
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for g in requested_genres(query).unwrap_or_default() {
|
|
||||||
if doc.subject_key.iter().any(|s| s.eq_ignore_ascii_case(&g.subject)) {
|
|
||||||
fields.insert(format!("genres/{}", g.name), "true".into());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Some(DiscoveryRecord {
|
Some(DiscoveryRecord {
|
||||||
upstream_id: UPSTREAM.into(),
|
upstream_id: UPSTREAM.into(),
|
||||||
record_id: id.to_string(),
|
record_id: id.to_string(),
|
||||||
@@ -324,13 +280,14 @@ async fn project(
|
|||||||
query: &GatewayQuery,
|
query: &GatewayQuery,
|
||||||
n: usize,
|
n: usize,
|
||||||
deadline: Duration,
|
deadline: Duration,
|
||||||
|
lenient: bool,
|
||||||
) -> Vec<DiscoveryRecord> {
|
) -> Vec<DiscoveryRecord> {
|
||||||
let mut seen = HashSet::new();
|
let mut seen = HashSet::new();
|
||||||
let candidates: Vec<SearchDoc> = docs
|
let candidates: Vec<SearchDoc> = docs
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|d| {
|
.filter(|d| {
|
||||||
d.work_id().is_some()
|
d.work_id().is_some()
|
||||||
&& d.cover_url().is_some()
|
&& (lenient || d.cover_url().is_some())
|
||||||
&& d.title.as_deref().is_some_and(|t| !t.trim().is_empty())
|
&& d.title.as_deref().is_some_and(|t| !t.trim().is_empty())
|
||||||
})
|
})
|
||||||
.filter(|d| seen.insert(d.key.clone()))
|
.filter(|d| seen.insert(d.key.clone()))
|
||||||
@@ -353,7 +310,7 @@ async fn project(
|
|||||||
debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped");
|
debug!(from = doc.key, to = w.id, "openlibrary: merged work; skipped");
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
to_record(&doc, w.description.as_deref()?, query)
|
to_record_with(&doc, w.description.as_deref(), query, lenient)
|
||||||
}
|
}
|
||||||
Ok(None) => None,
|
Ok(None) => None,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
@@ -504,34 +461,27 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else {
|
let Some(doc) = client.doc_by_work_id(id, SEARCH_DEADLINE).await? else {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
};
|
};
|
||||||
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE).await);
|
// Lenient: an explicit work lookup keeps the card even without a
|
||||||
|
// description, and finds a cover by ISBN when the work has none.
|
||||||
|
return Ok(project(client, vec![doc], query, 1, SEARCH_DEADLINE, true).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
// A curated row, narrowed to its genres when it names any.
|
// A curated row.
|
||||||
if let Some(kind) = browse_kind(query) {
|
if let Some(kind) = browse_kind(query) {
|
||||||
let Some(genres) = requested_genres(query) else {
|
|
||||||
debug!("openlibrary: unreadable genres filter; returning nothing");
|
|
||||||
return Ok(Vec::new());
|
|
||||||
};
|
|
||||||
let subjects: Vec<String> = genres.into_iter().map(|g| g.subject).collect();
|
|
||||||
let n = self.discovery_n.min(cap);
|
let n = self.discovery_n.min(cap);
|
||||||
let docs = match kind {
|
let docs = match kind {
|
||||||
Browse::Popular => {
|
Browse::Popular => {
|
||||||
client
|
client
|
||||||
.trending(
|
.trending((n * SEARCH_OVERFETCH).min(MAX_LIMIT), BROWSE_DEADLINE)
|
||||||
(n * SEARCH_OVERFETCH).min(MAX_LIMIT),
|
|
||||||
&subjects,
|
|
||||||
BROWSE_DEADLINE,
|
|
||||||
)
|
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
Browse::Fresh => {
|
Browse::Fresh => {
|
||||||
client
|
client
|
||||||
.fresh((n * FRESH_OVERFETCH).min(MAX_LIMIT), &subjects, BROWSE_DEADLINE)
|
.fresh((n * FRESH_OVERFETCH).min(MAX_LIMIT), BROWSE_DEADLINE)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
return Ok(project(client, docs, query, n, BROWSE_DEADLINE).await);
|
return Ok(project(client, docs, query, n, BROWSE_DEADLINE, false).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Free text.
|
// Free text.
|
||||||
@@ -546,7 +496,7 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
let docs = client
|
let docs = client
|
||||||
.search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE)
|
.search(text, (n * SEARCH_OVERFETCH).min(MAX_LIMIT), SEARCH_DEADLINE)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(project(client, docs, query, n, SEARCH_DEADLINE).await)
|
Ok(project(client, docs, query, n, SEARCH_DEADLINE, false).await)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A card has no bytes: the record IS the payload.
|
/// A card has no bytes: the record IS the payload.
|
||||||
@@ -565,7 +515,8 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
// different work (a merge) is not this cid's record.
|
// different work (a merge) is not this cid's record.
|
||||||
let record = match self.client.as_ref() {
|
let record = match self.client.as_ref() {
|
||||||
Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await {
|
Some(c) => match c.doc_by_work_id(id, SEARCH_DEADLINE).await {
|
||||||
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE)
|
// Lenient, like the `openlibraryid` lookup that minted this card.
|
||||||
|
Ok(Some(doc)) => project(c, vec![doc], &GatewayQuery::from_free_text(""), 1, SEARCH_DEADLINE, true)
|
||||||
.await
|
.await
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.find(|r| r.record_id == id),
|
.find(|r| r.record_id == id),
|
||||||
@@ -621,7 +572,7 @@ mod tests {
|
|||||||
cover_i: Some(14348537),
|
cover_i: Some(14348537),
|
||||||
first_publish_year: Some(1813),
|
first_publish_year: Some(1813),
|
||||||
id_project_gutenberg: vec![],
|
id_project_gutenberg: vec![],
|
||||||
subject_key: vec![],
|
isbn: vec![],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -728,6 +679,95 @@ mod tests {
|
|||||||
assert!(to_record(&edition, "A novel.", &q("", &[])).is_none());
|
assert!(to_record(&edition, "A novel.", &q("", &[])).is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_lenient_card_needs_only_a_title_and_finds_a_cover_by_isbn() {
|
||||||
|
let mut d = pride();
|
||||||
|
d.cover_i = None;
|
||||||
|
d.isbn = vec!["1955252092".into(), "9781955252096".into()];
|
||||||
|
// Strict still refuses it, whatever the description.
|
||||||
|
assert!(to_record_with(&d, Some("A novel."), &q("", &[]), false).is_none());
|
||||||
|
// Lenient mints it with the ISBN-13 cover and no description at all.
|
||||||
|
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
r.fields["poster_url"],
|
||||||
|
"https://covers.openlibrary.org/b/isbn/9781955252096-L.jpg?default=false"
|
||||||
|
);
|
||||||
|
assert!(!r.fields.keys().any(|k| k.starts_with("description")));
|
||||||
|
assert_eq!(r.fields["openlibraryid"], "OL66554W");
|
||||||
|
assert_eq!(r.fields["fileType"], "card");
|
||||||
|
// Same address as the strict card for the same work.
|
||||||
|
let strict = to_record(&pride(), "A novel.", &q("", &[])).unwrap();
|
||||||
|
let key = |r: &DiscoveryRecord| r.fields.keys().find(|k| k.starts_with("cids/")).cloned();
|
||||||
|
assert_eq!(key(&r), key(&strict));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_lenient_card_with_no_cover_anywhere_has_no_poster_rather_than_no_card() {
|
||||||
|
let mut d = pride();
|
||||||
|
d.cover_i = None;
|
||||||
|
let r = to_record_with(&d, None, &q("", &[]), true).unwrap();
|
||||||
|
assert!(!r.fields.contains_key("poster_url"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn isbn_cover_prefers_an_isbn13_and_ignores_junk() {
|
||||||
|
let mut d = SearchDoc::default();
|
||||||
|
d.isbn = vec!["not-an-isbn".into(), "0306406152".into(), "9780306406157".into()];
|
||||||
|
assert_eq!(
|
||||||
|
d.isbn_cover_url().as_deref(),
|
||||||
|
Some("https://covers.openlibrary.org/b/isbn/9780306406157-L.jpg?default=false")
|
||||||
|
);
|
||||||
|
d.isbn = vec!["0306406152".into()];
|
||||||
|
assert!(d.isbn_cover_url().unwrap().contains("0306406152"));
|
||||||
|
d.isbn = vec!["x".into()];
|
||||||
|
assert_eq!(d.isbn_cover_url(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_named_work_without_cover_or_description_still_gets_its_card_but_a_search_does_not() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
let doc = json!({
|
||||||
|
"key": "/works/OL39284292W",
|
||||||
|
"title": "Mother of Learning",
|
||||||
|
"author_name": ["nobody103"],
|
||||||
|
"first_publish_year": 2021,
|
||||||
|
"language": ["eng"],
|
||||||
|
"isbn": ["9781955252096"]
|
||||||
|
});
|
||||||
|
// The lookup by id and the free-text search both find the same index doc.
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"numFound": 1, "docs": [doc]})))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/works/OL39284292W.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||||
|
"key": "/works/OL39284292W", "title": "Mother of Learning"
|
||||||
|
})))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
let (p, _dir) = plugin_against(&server).await;
|
||||||
|
|
||||||
|
let by_id = p
|
||||||
|
.handle_query(
|
||||||
|
&q("", &[("openlibraryid", "OL39284292W"), ("contentKind", "book")]),
|
||||||
|
10,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(by_id.len(), 1, "the work page lookup yields a card");
|
||||||
|
assert_eq!(by_id[0].fields["title"], "Mother of Learning");
|
||||||
|
assert!(by_id[0].fields["poster_url"].contains("/b/isbn/9781955252096-L.jpg"));
|
||||||
|
assert!(!by_id[0].fields.keys().any(|k| k.starts_with("description")));
|
||||||
|
|
||||||
|
let by_search = p
|
||||||
|
.handle_query(&q("mother of learning", &[("fileType", "card"), ("contentKind", "book")]), 10)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(by_search.is_empty(), "search rows keep the strict bar");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn echoes_browse_markers_but_no_other_filter() {
|
fn echoes_browse_markers_but_no_other_filter() {
|
||||||
let r = to_record(
|
let r = to_record(
|
||||||
@@ -1051,109 +1091,6 @@ mod tests {
|
|||||||
server.verify().await;
|
server.verify().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn genre(subject: &str, name: &str) -> Genre {
|
|
||||||
Genre { subject: subject.into(), name: name.into() }
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn genre_slugs_map_to_open_library_subjects_and_card_names() {
|
|
||||||
assert_eq!(
|
|
||||||
requested_genres(&q("", &[("genres", "science-fiction")])),
|
|
||||||
Some(vec![genre("science_fiction", "Science Fiction")])
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
requested_genres(&q("", &[("genres", "Fantasy")])),
|
|
||||||
Some(vec![genre("fantasy", "Fantasy")])
|
|
||||||
);
|
|
||||||
let mut several = q("", &[]);
|
|
||||||
several.filters.insert(
|
|
||||||
"genres".into(),
|
|
||||||
vec!["fantasy".into(), "Fantasy".into(), "mystery".into()],
|
|
||||||
);
|
|
||||||
assert_eq!(
|
|
||||||
requested_genres(&several),
|
|
||||||
Some(vec![genre("fantasy", "Fantasy"), genre("mystery", "Mystery")])
|
|
||||||
);
|
|
||||||
assert_eq!(requested_genres(&q("", &[])), Some(vec![]));
|
|
||||||
// The subject is pasted into an Open Library query: anything but a
|
|
||||||
// plain slug is refused.
|
|
||||||
assert_eq!(requested_genres(&q("", &[("genres", "fantasy)OR(*")])), None);
|
|
||||||
assert_eq!(requested_genres(&q("", &[("genres", "--")])), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_card_names_only_the_requested_genres_open_library_files_it_under() {
|
|
||||||
let mut doc = pride();
|
|
||||||
doc.subject_key = vec!["fantasy".into(), "magic".into()];
|
|
||||||
|
|
||||||
let row = q("", &[("popular", "true"), ("genres", "fantasy")]);
|
|
||||||
let r = to_record(&doc, "A novel.", &row).unwrap();
|
|
||||||
assert_eq!(r.fields["genres/Fantasy"], "true");
|
|
||||||
assert_eq!(r.fields.keys().filter(|k| k.starts_with("genres/")).count(), 1);
|
|
||||||
assert!(meta_feeder_sdk::query_eval::record_matches(&r.fields, &row));
|
|
||||||
|
|
||||||
let other = q("", &[("popular", "true"), ("genres", "mystery")]);
|
|
||||||
let r = to_record(&doc, "A novel.", &other).unwrap();
|
|
||||||
assert!(!r.fields.keys().any(|k| k.starts_with("genres")));
|
|
||||||
assert!(!meta_feeder_sdk::query_eval::record_matches(&r.fields, &other));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn a_genre_row_reads_trending_search_narrowed_to_its_subject() {
|
|
||||||
let server = MockServer::start().await;
|
|
||||||
let mut dune = doc_json("OL893415W", "Dune", Some(11481354));
|
|
||||||
dune["subject_key"] = json!(["science_fiction", "fiction"]);
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/search.json"))
|
|
||||||
.and(query_param("q", "subject_key:(science_fiction)"))
|
|
||||||
.and(query_param("sort", "trending"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
|
||||||
"numFound": 1,
|
|
||||||
"docs": [dune]
|
|
||||||
})))
|
|
||||||
.expect(1)
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/trending/daily.json"))
|
|
||||||
.respond_with(ResponseTemplate::new(500))
|
|
||||||
.expect(0)
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
Mock::given(method("GET"))
|
|
||||||
.and(path("/works/OL893415W.json"))
|
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL893415W", "description": "A novel." })))
|
|
||||||
.mount(&server)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let (p, _dir) = plugin_against(&server).await;
|
|
||||||
// meta-read's Science Fiction row, as `config.rs` spells it.
|
|
||||||
let row = q(
|
|
||||||
"",
|
|
||||||
&[
|
|
||||||
("popular", "true"),
|
|
||||||
("domain", "literature"),
|
|
||||||
("contentKind", "book"),
|
|
||||||
("fileType", "card"),
|
|
||||||
("genres", "science-fiction"),
|
|
||||||
],
|
|
||||||
);
|
|
||||||
let out = p.handle_query(&row, 20).await.unwrap();
|
|
||||||
assert_eq!(out.len(), 1);
|
|
||||||
assert_eq!(out[0].fields["genres/Science Fiction"], "true");
|
|
||||||
// The gateway's filter fails closed on genres; it keeps this card.
|
|
||||||
assert!(meta_feeder_sdk::query_eval::record_matches(&out[0].fields, &row));
|
|
||||||
server.verify().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn an_unreadable_genre_answers_nothing_before_any_request() {
|
|
||||||
let (p, server, _dir) = plugin_that_must_not_call_out().await;
|
|
||||||
let row = q("", &[("popular", "true"), ("genres", "fantasy)OR(*")]);
|
|
||||||
assert!(p.handle_query(&row, 20).await.unwrap().is_empty());
|
|
||||||
server.verify().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn a_rate_limited_search_surfaces_as_rate_limited() {
|
async fn a_rate_limited_search_surfaces_as_rate_limited() {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ pub const DEFAULT_API_BASE: &str = "https://openlibrary.org";
|
|||||||
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
|
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
|
||||||
/// `poster_url` names, so one size is published, not three.
|
/// `poster_url` names, so one size is published, not three.
|
||||||
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
|
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
|
||||||
|
/// Cover by ISBN. Open Library often holds an *edition's* cover for a work whose
|
||||||
|
/// own `covers` list is empty.
|
||||||
|
const COVERS_ISBN_BASE: &str = "https://covers.openlibrary.org/b/isbn";
|
||||||
|
|
||||||
/// Config key (and env seed) for the operator contact sent in the User-Agent.
|
/// Config key (and env seed) for the operator contact sent in the User-Agent.
|
||||||
pub const CONTACT_KEY: &str = "contact";
|
pub const CONTACT_KEY: &str = "contact";
|
||||||
@@ -58,12 +61,9 @@ const RETRY_BACKOFF_MS: u64 = 250;
|
|||||||
/// The field selection for every doc-shaped request, written once so a card
|
/// The field selection for every doc-shaped request, written once so a card
|
||||||
/// resolved by search and the same card resolved by id are byte-identical.
|
/// resolved by search and the same card resolved by id are byte-identical.
|
||||||
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
|
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
|
||||||
|
/// A lookup of one named work also asks for its ISBNs, so a cover can be found by
|
||||||
/// [`SEARCH_FIELDS`] plus the work's subject keys, read only by a row narrowed
|
/// ISBN. Not on the shared list: a search row would carry hundreds of ISBNs per doc.
|
||||||
/// to subjects: a card names a genre only when Open Library files the work
|
const DOC_BY_ID_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language,isbn";
|
||||||
/// under it.
|
|
||||||
const SUBJECT_SEARCH_FIELDS: &str =
|
|
||||||
"key,title,author_name,cover_i,first_publish_year,language,subject_key";
|
|
||||||
|
|
||||||
/// Cache lifetimes.
|
/// Cache lifetimes.
|
||||||
///
|
///
|
||||||
@@ -205,10 +205,10 @@ pub struct SearchDoc {
|
|||||||
pub first_publish_year: Option<i32>,
|
pub first_publish_year: Option<i32>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub id_project_gutenberg: Vec<String>,
|
pub id_project_gutenberg: Vec<String>,
|
||||||
/// Only requested by a subject-narrowed row ([`SUBJECT_SEARCH_FIELDS`]);
|
/// ISBNs of the work's editions. Only requested for an `openlibraryid`
|
||||||
/// empty everywhere else.
|
/// lookup (see [`DOC_BY_ID_FIELDS`]); empty everywhere else.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub subject_key: Vec<String>,
|
pub isbn: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SearchDoc {
|
impl SearchDoc {
|
||||||
@@ -224,6 +224,22 @@ impl SearchDoc {
|
|||||||
.filter(|c| *c > 0)
|
.filter(|c| *c > 0)
|
||||||
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
|
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A cover looked up by one of the work's ISBNs (an ISBN-13 when there is one),
|
||||||
|
/// for a work with no cover of its own.
|
||||||
|
///
|
||||||
|
/// `?default=false` makes Open Library answer 404 when it has no image for the
|
||||||
|
/// ISBN, instead of a 1×1 placeholder that would be seeded as the poster.
|
||||||
|
pub fn isbn_cover_url(&self) -> Option<String> {
|
||||||
|
let digits_only = |s: &&String| s.chars().all(|c| c.is_ascii_digit());
|
||||||
|
let isbn = self
|
||||||
|
.isbn
|
||||||
|
.iter()
|
||||||
|
.filter(digits_only)
|
||||||
|
.find(|s| s.len() == 13)
|
||||||
|
.or_else(|| self.isbn.iter().filter(digits_only).find(|s| s.len() == 10))?;
|
||||||
|
Some(format!("{COVERS_ISBN_BASE}/{isbn}-L.jpg?default=false"))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What a work JSON contributes to a card.
|
/// What a work JSON contributes to a card.
|
||||||
@@ -388,33 +404,11 @@ impl OpenLibraryClient {
|
|||||||
/// The `popular:true` row: what Open Library readers opened today.
|
/// The `popular:true` row: what Open Library readers opened today.
|
||||||
///
|
///
|
||||||
/// The ordering is a popularity signal, but no count is read or stored.
|
/// The ordering is a popularity signal, but no count is read or stored.
|
||||||
///
|
|
||||||
/// `subjects` narrows the row to works filed under any of them (a genre
|
|
||||||
/// row). `/trending/daily.json` takes no subject, so a narrowed row asks
|
|
||||||
/// `search.json` sorted `trending`, the same signal over a filtered set.
|
|
||||||
pub async fn trending(
|
pub async fn trending(
|
||||||
&self,
|
&self,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
subjects: &[String],
|
|
||||||
deadline: Duration,
|
deadline: Duration,
|
||||||
) -> Result<Vec<SearchDoc>, GatewayError> {
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
if !subjects.is_empty() {
|
|
||||||
let body = self
|
|
||||||
.get_cached(
|
|
||||||
&format!("openlibrary:trending:subjects:{}:{limit}", subjects.join("+")),
|
|
||||||
BROWSE_TTL_SECS,
|
|
||||||
"/search.json",
|
|
||||||
&[
|
|
||||||
("q", subject_clause(subjects)),
|
|
||||||
("sort", "trending".into()),
|
|
||||||
("fields", SUBJECT_SEARCH_FIELDS.into()),
|
|
||||||
("limit", limit.to_string()),
|
|
||||||
],
|
|
||||||
deadline,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
return Ok(parse_docs(body.get("docs")));
|
|
||||||
}
|
|
||||||
let body = self
|
let body = self
|
||||||
.get_cached(
|
.get_cached(
|
||||||
&format!("openlibrary:trending:daily:{limit}"),
|
&format!("openlibrary:trending:daily:{limit}"),
|
||||||
@@ -434,30 +428,21 @@ impl OpenLibraryClient {
|
|||||||
/// over a broad query is dominated by coverless catalogue stubs. The year
|
/// over a broad query is dominated by coverless catalogue stubs. The year
|
||||||
/// window keeps the row about new books; the card gate (cover + description)
|
/// window keeps the row about new books; the card gate (cover + description)
|
||||||
/// drops the stubs that remain, so the caller over-fetches.
|
/// drops the stubs that remain, so the caller over-fetches.
|
||||||
///
|
|
||||||
/// `subjects` narrows the row as in [`Self::trending`].
|
|
||||||
pub async fn fresh(
|
pub async fn fresh(
|
||||||
&self,
|
&self,
|
||||||
limit: usize,
|
limit: usize,
|
||||||
subjects: &[String],
|
|
||||||
deadline: Duration,
|
deadline: Duration,
|
||||||
) -> Result<Vec<SearchDoc>, GatewayError> {
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
let year = 1970 + now_secs() / 31_556_952;
|
let year = 1970 + now_secs() / 31_556_952;
|
||||||
let mut q = format!("first_publish_year:[{} TO {year}]", year - 1);
|
|
||||||
let mut fields = SEARCH_FIELDS;
|
|
||||||
if !subjects.is_empty() {
|
|
||||||
q = format!("{q} AND {}", subject_clause(subjects));
|
|
||||||
fields = SUBJECT_SEARCH_FIELDS;
|
|
||||||
}
|
|
||||||
let body = self
|
let body = self
|
||||||
.get_cached(
|
.get_cached(
|
||||||
&format!("openlibrary:fresh:{year}:{}:{limit}", subjects.join("+")),
|
&format!("openlibrary:fresh:{year}:{limit}"),
|
||||||
BROWSE_TTL_SECS,
|
BROWSE_TTL_SECS,
|
||||||
"/search.json",
|
"/search.json",
|
||||||
&[
|
&[
|
||||||
("q", q),
|
("q", format!("first_publish_year:[{} TO {year}]", year - 1)),
|
||||||
("sort", "new".into()),
|
("sort", "new".into()),
|
||||||
("fields", fields.into()),
|
("fields", SEARCH_FIELDS.into()),
|
||||||
("limit", limit.to_string()),
|
("limit", limit.to_string()),
|
||||||
],
|
],
|
||||||
deadline,
|
deadline,
|
||||||
@@ -481,12 +466,13 @@ impl OpenLibraryClient {
|
|||||||
) -> Result<Option<SearchDoc>, GatewayError> {
|
) -> Result<Option<SearchDoc>, GatewayError> {
|
||||||
let body = self
|
let body = self
|
||||||
.get_cached(
|
.get_cached(
|
||||||
&format!("openlibrary:key:{id}"),
|
// `key2`: entries cached before ISBNs were requested lack them.
|
||||||
|
&format!("openlibrary:key2:{id}"),
|
||||||
WORK_TTL_SECS,
|
WORK_TTL_SECS,
|
||||||
"/search.json",
|
"/search.json",
|
||||||
&[
|
&[
|
||||||
("q", format!("key:/works/{id}")),
|
("q", format!("key:/works/{id}")),
|
||||||
("fields", SEARCH_FIELDS.into()),
|
("fields", DOC_BY_ID_FIELDS.into()),
|
||||||
("limit", "1".into()),
|
("limit", "1".into()),
|
||||||
],
|
],
|
||||||
deadline,
|
deadline,
|
||||||
@@ -534,7 +520,7 @@ impl OpenLibraryClient {
|
|||||||
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
|
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
|
||||||
first_publish_year: None,
|
first_publish_year: None,
|
||||||
id_project_gutenberg: Vec::new(),
|
id_project_gutenberg: Vec::new(),
|
||||||
subject_key: Vec::new(),
|
isbn: Vec::new(),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -656,248 +642,6 @@ impl OpenLibraryClient {
|
|||||||
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
Ok(work)
|
Ok(work)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// What Open Library records about an etext: the one work that lists it,
|
|
||||||
/// several works, or none.
|
|
||||||
///
|
|
||||||
/// Same lookup as [`Self::gutenberg_work`], but it keeps the difference
|
|
||||||
/// between "several" and "none", which that one folds into `None`: only an
|
|
||||||
/// etext no work lists may be bound by title ([`Self::title_work`]). Several
|
|
||||||
/// works listing one etext is an unmerged duplicate, and guessing between
|
|
||||||
/// them is exactly what the identifier rule refuses.
|
|
||||||
pub async fn gutenberg_etext(
|
|
||||||
&self,
|
|
||||||
gutenberg_id: u64,
|
|
||||||
deadline: Duration,
|
|
||||||
) -> Result<Etext, GatewayError> {
|
|
||||||
let key = format!("openlibrary:etext:{gutenberg_id}");
|
|
||||||
if let Some(c) = self.cache_get(&key) {
|
|
||||||
if let Some(etext) = Etext::from_cached(&c.body) {
|
|
||||||
let ttl = if matches!(etext, Etext::One(_)) {
|
|
||||||
BINDING_HIT_TTL_SECS
|
|
||||||
} else {
|
|
||||||
BINDING_MISS_TTL_SECS
|
|
||||||
};
|
|
||||||
if now_secs().saturating_sub(c.at) <= ttl {
|
|
||||||
return Ok(etext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let body = self
|
|
||||||
.get(
|
|
||||||
"/search.json",
|
|
||||||
&[
|
|
||||||
("q", format!("id_project_gutenberg:{gutenberg_id}")),
|
|
||||||
("fields", "key,id_project_gutenberg".into()),
|
|
||||||
("limit", "2".into()),
|
|
||||||
],
|
|
||||||
deadline,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let etext = etext_of(&body, gutenberg_id);
|
|
||||||
self.cache_put(&key, &etext.to_cached());
|
|
||||||
Ok(etext)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The work a book is bound to by **title and author**, for an etext Open
|
|
||||||
/// Library lists under no work.
|
|
||||||
///
|
|
||||||
/// Open Library's `id_project_gutenberg` links are patchy even for the most
|
|
||||||
/// read books — none of the editions of *Frankenstein; or, The Modern
|
|
||||||
/// Prometheus* (OL450063W) lists etext 84 — so the identifier rule alone left
|
|
||||||
/// them unbound, and their book pages, which ask for `openlibraryid:OL…W`,
|
|
||||||
/// empty. This is the weaker proof the trust model already has a grade for:
|
|
||||||
/// the caller stamps `anchorMethod=title`, the gateway records
|
|
||||||
/// `matchedBy/openlibrary:OL…W`, and meta-read ranks such an edition after an
|
|
||||||
/// identifier-bound one.
|
|
||||||
///
|
|
||||||
/// Same main title ([`main_title`]), an author with the same surname, and,
|
|
||||||
/// among those, the work with the most editions — Open Library holds many
|
|
||||||
/// small works titled *Frankenstein* by Mary Shelley (adaptations, annotated
|
|
||||||
/// and school editions), and the canonical one is the one most published.
|
|
||||||
/// `None` when nothing matches or two works tie.
|
|
||||||
pub async fn title_work(
|
|
||||||
&self,
|
|
||||||
title: &str,
|
|
||||||
author: &str,
|
|
||||||
deadline: Duration,
|
|
||||||
) -> Result<Option<String>, GatewayError> {
|
|
||||||
let (want, surname) = (main_title(title), surname(author));
|
|
||||||
if want.is_empty() || surname.is_empty() {
|
|
||||||
return Ok(None);
|
|
||||||
}
|
|
||||||
let key = format!("openlibrary:titlework:{want}:{surname}");
|
|
||||||
if let Some(c) = self.cache_get(&key) {
|
|
||||||
let work = c.body.get("work").and_then(Value::as_str).map(str::to_string);
|
|
||||||
let ttl = if work.is_some() {
|
|
||||||
BINDING_HIT_TTL_SECS
|
|
||||||
} else {
|
|
||||||
BINDING_MISS_TTL_SECS
|
|
||||||
};
|
|
||||||
if now_secs().saturating_sub(c.at) <= ttl {
|
|
||||||
return Ok(work);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let body = self
|
|
||||||
.get(
|
|
||||||
"/search.json",
|
|
||||||
&[
|
|
||||||
("title", want.clone()),
|
|
||||||
("author", surname.clone()),
|
|
||||||
("fields", "key,title,author_name,edition_count".into()),
|
|
||||||
("limit", "20".into()),
|
|
||||||
],
|
|
||||||
deadline,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
let work = best_title_work(&body, &want, &surname);
|
|
||||||
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
|
||||||
Ok(work)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// What Open Library records about one Gutenberg etext. See
|
|
||||||
/// [`OpenLibraryClient::gutenberg_etext`].
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
||||||
pub enum Etext {
|
|
||||||
/// Exactly one work lists it: an identifier binding, graded Anchored.
|
|
||||||
One(String),
|
|
||||||
/// Several works list it: an unmerged duplicate, so no binding at all.
|
|
||||||
Several,
|
|
||||||
/// No work lists it: the one case a title binding may fill.
|
|
||||||
Unlisted,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl Etext {
|
|
||||||
fn to_cached(&self) -> Value {
|
|
||||||
match self {
|
|
||||||
Etext::One(work) => serde_json::json!({ "etext": "one", "work": work }),
|
|
||||||
Etext::Several => serde_json::json!({ "etext": "several" }),
|
|
||||||
Etext::Unlisted => serde_json::json!({ "etext": "unlisted" }),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn from_cached(v: &Value) -> Option<Self> {
|
|
||||||
match v.get("etext")?.as_str()? {
|
|
||||||
"one" => Some(Etext::One(v.get("work")?.as_str()?.to_string())),
|
|
||||||
"several" => Some(Etext::Several),
|
|
||||||
"unlisted" => Some(Etext::Unlisted),
|
|
||||||
_ => None,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn etext_of(body: &Value, gutenberg_id: u64) -> Etext {
|
|
||||||
if let Some(work) = single_gutenberg_work(body, gutenberg_id) {
|
|
||||||
return Etext::One(work);
|
|
||||||
}
|
|
||||||
let docs = parse_docs(body.get("docs"));
|
|
||||||
let total = body
|
|
||||||
.get("numFound")
|
|
||||||
.or_else(|| body.get("num_found"))
|
|
||||||
.and_then(Value::as_u64)
|
|
||||||
.unwrap_or(docs.len() as u64);
|
|
||||||
if total == 0 && docs.is_empty() {
|
|
||||||
Etext::Unlisted
|
|
||||||
} else {
|
|
||||||
Etext::Several
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The part of a title two catalogues agree on: before any subtitle (`;`, `:`,
|
|
||||||
/// a bracket, or `, or`), lower-cased, letters and digits only. Gutenberg's
|
|
||||||
/// *Frankenstein; Or, The Modern Prometheus* and Open Library's *Frankenstein;
|
|
||||||
/// or, The Modern Prometheus* are both `frankenstein`.
|
|
||||||
pub fn main_title(raw: &str) -> String {
|
|
||||||
let lower = raw.to_lowercase();
|
|
||||||
let head = lower
|
|
||||||
.split([';', ':', '(', '['])
|
|
||||||
.next()
|
|
||||||
.unwrap_or_default();
|
|
||||||
let head = head.find(", or").map_or(head, |i| &head[..i]);
|
|
||||||
head.split(|c: char| !c.is_alphanumeric())
|
|
||||||
.filter(|w| !w.is_empty())
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join(" ")
|
|
||||||
}
|
|
||||||
|
|
||||||
/// The surname a name ends on, lower-cased, letters and digits only: Gutendex
|
|
||||||
/// writes `Shelley, Mary Wollstonecraft`, Open Library `Mary Shelley`; both give
|
|
||||||
/// `shelley`.
|
|
||||||
pub fn surname(raw: &str) -> String {
|
|
||||||
let raw = raw.trim();
|
|
||||||
let last = match raw.split_once(',') {
|
|
||||||
Some((last, _)) => last,
|
|
||||||
None => raw.rsplit(char::is_whitespace).next().unwrap_or_default(),
|
|
||||||
};
|
|
||||||
last.chars()
|
|
||||||
.filter(|c| c.is_alphanumeric())
|
|
||||||
.collect::<String>()
|
|
||||||
.to_lowercase()
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Does `name` carry `surname`? Compared with the separators removed, so a
|
|
||||||
/// compound surname (`Saint-Exupéry`) matches however it is spaced.
|
|
||||||
fn author_has_surname(name: &str, surname: &str) -> bool {
|
|
||||||
let flat = name
|
|
||||||
.chars()
|
|
||||||
.filter(|c| c.is_alphanumeric())
|
|
||||||
.collect::<String>()
|
|
||||||
.to_lowercase();
|
|
||||||
!surname.is_empty() && flat.contains(surname)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One document of the title search, with the edition count the choice rests on.
|
|
||||||
#[derive(Debug, Deserialize)]
|
|
||||||
struct TitleDoc {
|
|
||||||
#[serde(default)]
|
|
||||||
key: String,
|
|
||||||
#[serde(default)]
|
|
||||||
title: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
author_name: Vec<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
edition_count: Option<u64>,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Decide a title binding from a `title=…&author=…` response. See
|
|
||||||
/// [`OpenLibraryClient::title_work`].
|
|
||||||
fn best_title_work(body: &Value, want: &str, surname: &str) -> Option<String> {
|
|
||||||
let docs: Vec<TitleDoc> = body
|
|
||||||
.get("docs")
|
|
||||||
.and_then(Value::as_array)
|
|
||||||
.map(|a| {
|
|
||||||
a.iter()
|
|
||||||
.filter_map(|d| serde_json::from_value::<TitleDoc>(d.clone()).ok())
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.unwrap_or_default();
|
|
||||||
let mut best: Option<(u64, String)> = None;
|
|
||||||
let mut tie = false;
|
|
||||||
for d in docs {
|
|
||||||
let Some(id) = d.key.strip_prefix("/works/").filter(|id| is_work_id(id)) else {
|
|
||||||
continue;
|
|
||||||
};
|
|
||||||
if main_title(d.title.as_deref().unwrap_or_default()) != want {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if !d.author_name.iter().any(|a| author_has_surname(a, surname)) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let n = d.edition_count.unwrap_or(0);
|
|
||||||
let top = best.as_ref().map(|(m, _)| *m);
|
|
||||||
if top.is_none_or(|m| n > m) {
|
|
||||||
best = Some((n, id.to_string()));
|
|
||||||
tie = false;
|
|
||||||
} else if top == Some(n) {
|
|
||||||
tie = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if tie {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
best.map(|(_, id)| id)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
||||||
@@ -922,12 +666,6 @@ fn single_gutenberg_work(body: &Value, gutenberg_id: u64) -> Option<String> {
|
|||||||
doc.work_id().map(str::to_string)
|
doc.work_id().map(str::to_string)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `subject_key:(a OR b)`. The keys come from [`crate::openlibrary`]'s genre
|
|
||||||
/// parser, which admits only `[a-z0-9_]`, so nothing here needs escaping.
|
|
||||||
fn subject_clause(subjects: &[String]) -> String {
|
|
||||||
format!("subject_key:({})", subjects.join(" OR "))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_docs(v: Option<&Value>) -> Vec<SearchDoc> {
|
fn parse_docs(v: Option<&Value>) -> Vec<SearchDoc> {
|
||||||
v.and_then(Value::as_array)
|
v.and_then(Value::as_array)
|
||||||
.map(|a| {
|
.map(|a| {
|
||||||
@@ -960,70 +698,6 @@ fn clean_description(raw: &str) -> Option<String> {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_main_title_drops_the_subtitle_case_and_punctuation() {
|
|
||||||
assert_eq!(main_title("Frankenstein; Or, The Modern Prometheus"), "frankenstein");
|
|
||||||
assert_eq!(main_title("Frankenstein; or, The Modern Prometheus"), "frankenstein");
|
|
||||||
assert_eq!(main_title("Frankenstein, or the Modern Prometheus"), "frankenstein");
|
|
||||||
assert_eq!(main_title("Frankenstein (Oxford Playscripts)"), "frankenstein");
|
|
||||||
assert_eq!(main_title("Alice's Adventures in Wonderland"), "alice s adventures in wonderland");
|
|
||||||
assert_eq!(main_title(""), "");
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_surname_reads_both_name_orders() {
|
|
||||||
assert_eq!(surname("Shelley, Mary Wollstonecraft"), "shelley");
|
|
||||||
assert_eq!(surname("Mary Shelley"), "shelley");
|
|
||||||
assert_eq!(surname(" Austen, Jane "), "austen");
|
|
||||||
assert_eq!(surname("Saint-Exupéry, Antoine de"), "saintexupéry");
|
|
||||||
assert!(author_has_surname("Antoine de Saint-Exupéry", "saintexupéry"));
|
|
||||||
assert!(!author_has_surname("Jason Cobley", "shelley"));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn title_doc(key: &str, title: &str, author: &str, editions: u64) -> Value {
|
|
||||||
serde_json::json!({ "key": key, "title": title, "author_name": [author], "edition_count": editions })
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn the_title_binding_takes_the_most_published_work_of_that_title_and_author() {
|
|
||||||
let body = serde_json::json!({ "docs": [
|
|
||||||
title_doc("/works/OL26860869W", "Frankenstein", "Mary Shelley", 84),
|
|
||||||
title_doc("/works/OL450063W", "Frankenstein; or, The Modern Prometheus", "Mary Shelley", 2187),
|
|
||||||
title_doc("/works/OL25595002W", "Mary Shelley's Frankenstein", "Mary Shelley", 9000),
|
|
||||||
title_doc("/works/OL15443537W", "Frankenstein", "Dean Koontz", 5000),
|
|
||||||
]});
|
|
||||||
// The 9000-edition work has another title, the 5000 one another author.
|
|
||||||
assert_eq!(best_title_work(&body, "frankenstein", "shelley").as_deref(), Some("OL450063W"));
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn a_tie_or_no_match_binds_nothing() {
|
|
||||||
let tie = serde_json::json!({ "docs": [
|
|
||||||
title_doc("/works/OL1W", "Frankenstein", "Mary Shelley", 10),
|
|
||||||
title_doc("/works/OL2W", "Frankenstein", "Mary Shelley", 10),
|
|
||||||
]});
|
|
||||||
assert_eq!(best_title_work(&tie, "frankenstein", "shelley"), None);
|
|
||||||
let other = serde_json::json!({ "docs": [title_doc("/works/OL3W", "Dracula", "Bram Stoker", 99)] });
|
|
||||||
assert_eq!(best_title_work(&other, "frankenstein", "shelley"), None);
|
|
||||||
let not_a_work = serde_json::json!({ "docs": [title_doc("/books/OL4M", "Frankenstein", "Mary Shelley", 99)] });
|
|
||||||
assert_eq!(best_title_work(¬_a_work, "frankenstein", "shelley"), None);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn an_etext_is_one_several_or_unlisted() {
|
|
||||||
let one = serde_json::json!({ "numFound": 1, "docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }] });
|
|
||||||
assert_eq!(etext_of(&one, 1342), Etext::One("OL66554W".into()));
|
|
||||||
let several = serde_json::json!({ "numFound": 2, "docs": [
|
|
||||||
{ "key": "/works/OL450063W", "id_project_gutenberg": ["84"] },
|
|
||||||
{ "key": "/works/OL18388W", "id_project_gutenberg": ["84"] }
|
|
||||||
]});
|
|
||||||
assert_eq!(etext_of(&several, 84), Etext::Several);
|
|
||||||
assert_eq!(etext_of(&serde_json::json!({ "numFound": 0, "docs": [] }), 84), Etext::Unlisted);
|
|
||||||
for e in [Etext::One("OL1W".into()), Etext::Several, Etext::Unlisted] {
|
|
||||||
assert_eq!(Etext::from_cached(&e.to_cached()), Some(e));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn work_ids_are_works_only() {
|
fn work_ids_are_works_only() {
|
||||||
assert!(is_work_id("OL66554W"));
|
assert!(is_work_id("OL66554W"));
|
||||||
@@ -1052,15 +726,6 @@ mod tests {
|
|||||||
assert_eq!(clean_description(" \r\n "), None);
|
assert_eq!(clean_description(" \r\n "), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn subject_clause_ors_the_keys() {
|
|
||||||
assert_eq!(subject_clause(&["fantasy".to_string()]), "subject_key:(fantasy)");
|
|
||||||
assert_eq!(
|
|
||||||
subject_clause(&["fantasy".to_string(), "mystery".to_string()]),
|
|
||||||
"subject_key:(fantasy OR mystery)"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn cover_minus_one_is_no_cover() {
|
fn cover_minus_one_is_no_cover() {
|
||||||
let mut d = SearchDoc {
|
let mut d = SearchDoc {
|
||||||
|
|||||||
Reference in New Issue
Block a user