openlibrary: genre rows read the subject and stamp the genre
Test / cargo test (push) Successful in 3m58s
Test / cargo test (push) Successful in 3m58s
The home rows send popular:true genres:<slug>. The card feeder had no genre handling (it served trending/daily), and the gateway drops a record without a genres/<Name> key when a genres filter is present, so every genre row came back empty. A popular row with genres now searches subject:"<subject>" sorted by readinglog (the same reader-shelving signal as trending, cut by genre) and stamps genres/<Name> on each card. Up to three genres per row, deduplicated, the trending path unchanged.
This commit is contained in:
@@ -106,6 +106,55 @@ fn browse_kind(q: &GatewayQuery) -> Option<Browse> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Most genres a single row may name; each costs one Open Library search.
|
||||
const MAX_GENRES: usize = 3;
|
||||
|
||||
/// The genre slugs a row asks for (`genres:fantasy`, `genres:science-fiction`).
|
||||
fn genre_slugs(q: &GatewayQuery) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for v in q.filters.get("genres").into_iter().flatten() {
|
||||
let slug = v.trim().to_lowercase();
|
||||
if !slug.is_empty() && !out.contains(&slug) {
|
||||
out.push(slug);
|
||||
}
|
||||
}
|
||||
out.truncate(MAX_GENRES);
|
||||
out
|
||||
}
|
||||
|
||||
/// The Open Library subject a slug names. Most slugs are the subject with
|
||||
/// dashes for spaces; the few that are not are spelled out.
|
||||
fn genre_subject(slug: &str) -> String {
|
||||
match slug {
|
||||
"mystery" | "detective" => "mystery and detective stories".into(),
|
||||
"scifi" | "sci-fi" => "science fiction".into(),
|
||||
other => other
|
||||
.chars()
|
||||
.map(|c| if c.is_alphanumeric() { c } else { ' ' })
|
||||
.collect::<String>()
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
}
|
||||
}
|
||||
|
||||
/// The display name a slug is stored under: `genres/Science Fiction`. The
|
||||
/// gateway compares both sides folded to lowercase alphanumerics, so
|
||||
/// `science-fiction` and `Science Fiction` are the same genre.
|
||||
fn genre_label(slug: &str) -> String {
|
||||
slug.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| !w.is_empty())
|
||||
.map(|w| {
|
||||
let mut cs = w.chars();
|
||||
match cs.next() {
|
||||
Some(f) => f.to_uppercase().collect::<String>() + cs.as_str(),
|
||||
None => String::new(),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
}
|
||||
|
||||
/// The browse markers a card echoes. See [`to_record`].
|
||||
const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"];
|
||||
|
||||
@@ -443,6 +492,35 @@ impl FeederPlugin for OpenLibraryPlugin {
|
||||
// A curated row.
|
||||
if let Some(kind) = browse_kind(query) {
|
||||
let n = self.discovery_n.min(cap);
|
||||
// A genre row: `trending/daily` has no subject to cut by, and the
|
||||
// gateway drops any card without `genres/<Name>` (the filter fails
|
||||
// closed), so the row came back empty. Ask Open Library for the
|
||||
// subject instead and stamp the genre the row asked for.
|
||||
let genres = genre_slugs(query);
|
||||
if kind == Browse::Popular && !genres.is_empty() {
|
||||
let mut out: Vec<DiscoveryRecord> = Vec::new();
|
||||
for slug in &genres {
|
||||
let docs = client
|
||||
.by_genre(
|
||||
&genre_subject(slug),
|
||||
(n * SEARCH_OVERFETCH).min(MAX_LIMIT),
|
||||
BROWSE_DEADLINE,
|
||||
)
|
||||
.await?;
|
||||
let label = genre_label(slug);
|
||||
for mut r in project(client, docs, query, n, BROWSE_DEADLINE).await {
|
||||
r.fields.insert(format!("genres/{label}"), "true".into());
|
||||
match out.iter_mut().find(|o| o.record_id == r.record_id) {
|
||||
Some(have) => have.fields.extend(
|
||||
r.fields.into_iter().filter(|(k, _)| k.starts_with("genres/")),
|
||||
),
|
||||
None => out.push(r),
|
||||
}
|
||||
}
|
||||
}
|
||||
out.truncate(n);
|
||||
return Ok(out);
|
||||
}
|
||||
let docs = match kind {
|
||||
Browse::Popular => {
|
||||
client
|
||||
@@ -948,6 +1026,59 @@ mod tests {
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_genre_row_reads_the_subject_and_stamps_the_genre_it_asked_for() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/search.json"))
|
||||
.and(query_param("q", "subject:\"science fiction\""))
|
||||
.and(query_param("sort", "readinglog"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"numFound": 1,
|
||||
"docs": [doc_json("OL66554W", "Nineteen Eighty-Four", Some(14348537))]
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/trending/daily.json"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(0)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/works/OL66554W.json"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL66554W", "description": "A novel." })))
|
||||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let (p, _dir) = plugin_against(&server).await;
|
||||
let row = q(
|
||||
"",
|
||||
&[("domain", "literature"), ("contentKind", "book"), ("popular", "true"), ("genres", "science-fiction")],
|
||||
);
|
||||
let out = p.handle_query(&row, 20).await.unwrap();
|
||||
assert_eq!(out.len(), 1);
|
||||
assert_eq!(out[0].fields["genres/Science Fiction"], "true");
|
||||
assert_eq!(out[0].fields["popular"], "true");
|
||||
// What the gateway checks before it keeps the card.
|
||||
assert!(meta_feeder_sdk::query_eval::record_matches(&out[0].fields, &row));
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn genre_slugs_map_to_subjects_and_to_stored_names() {
|
||||
assert_eq!(genre_subject("fantasy"), "fantasy");
|
||||
assert_eq!(genre_subject("science-fiction"), "science fiction");
|
||||
assert_eq!(genre_subject("mystery"), "mystery and detective stories");
|
||||
assert_eq!(genre_subject("a\"b"), "a b", "a quote must not reach the search syntax");
|
||||
assert_eq!(genre_label("science-fiction"), "Science Fiction");
|
||||
assert_eq!(genre_label("fantasy"), "Fantasy");
|
||||
let mut row = q("", &[("popular", "true")]);
|
||||
row.filters.insert("genres".into(), vec!["Fantasy".into(), "fantasy".into(), " ".into()]);
|
||||
assert_eq!(genre_slugs(&row), vec!["fantasy".to_string()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_fresh_row_sorts_search_by_new() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -395,6 +395,37 @@ impl OpenLibraryClient {
|
||||
Ok(parse_docs(body.get("works")))
|
||||
}
|
||||
|
||||
/// A genre row (`popular:true genres:<slug>`): the works Open Library readers
|
||||
/// shelve most under one subject, most-shelved first.
|
||||
///
|
||||
/// `trending/daily` carries no subject, so a genre row cannot be cut from it.
|
||||
/// `search.json` filtered by `subject:` and sorted by `readinglog` is the
|
||||
/// same signal (reader shelving) scoped to a genre; as with
|
||||
/// [`Self::trending`], the ordering is used and no count is read or stored.
|
||||
pub async fn by_genre(
|
||||
&self,
|
||||
subject: &str,
|
||||
limit: usize,
|
||||
deadline: Duration,
|
||||
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||
let subject = subject.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase();
|
||||
let body = self
|
||||
.get_cached(
|
||||
&format!("openlibrary:genre:{limit}:{subject}"),
|
||||
BROWSE_TTL_SECS,
|
||||
"/search.json",
|
||||
&[
|
||||
("q", format!("subject:\"{subject}\"")),
|
||||
("sort", "readinglog".into()),
|
||||
("fields", SEARCH_FIELDS.into()),
|
||||
("limit", limit.to_string()),
|
||||
],
|
||||
deadline,
|
||||
)
|
||||
.await?;
|
||||
Ok(parse_docs(body.get("docs")))
|
||||
}
|
||||
|
||||
/// The `fresh:true` row: recently added works first published this year or
|
||||
/// last.
|
||||
///
|
||||
|
||||
Reference in New Issue
Block a user