openlibrary: genre rows read the subject and stamp the genre
Test / cargo test (push) Successful in 3m58s
Test / cargo test (push) Successful in 3m58s
The home rows send popular:true genres:<slug>. The card feeder had no genre handling (it served trending/daily), and the gateway drops a record without a genres/<Name> key when a genres filter is present, so every genre row came back empty. A popular row with genres now searches subject:"<subject>" sorted by readinglog (the same reader-shelving signal as trending, cut by genre) and stamps genres/<Name> on each card. Up to three genres per row, deduplicated, the trending path unchanged.
This commit is contained in:
@@ -106,6 +106,55 @@ fn browse_kind(q: &GatewayQuery) -> Option<Browse> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Most genres a single row may name; each costs one Open Library search.
|
||||||
|
const MAX_GENRES: usize = 3;
|
||||||
|
|
||||||
|
/// The genre slugs a row asks for (`genres:fantasy`, `genres:science-fiction`).
|
||||||
|
fn genre_slugs(q: &GatewayQuery) -> Vec<String> {
|
||||||
|
let mut out: Vec<String> = Vec::new();
|
||||||
|
for v in q.filters.get("genres").into_iter().flatten() {
|
||||||
|
let slug = v.trim().to_lowercase();
|
||||||
|
if !slug.is_empty() && !out.contains(&slug) {
|
||||||
|
out.push(slug);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.truncate(MAX_GENRES);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The Open Library subject a slug names. Most slugs are the subject with
|
||||||
|
/// dashes for spaces; the few that are not are spelled out.
|
||||||
|
fn genre_subject(slug: &str) -> String {
|
||||||
|
match slug {
|
||||||
|
"mystery" | "detective" => "mystery and detective stories".into(),
|
||||||
|
"scifi" | "sci-fi" => "science fiction".into(),
|
||||||
|
other => other
|
||||||
|
.chars()
|
||||||
|
.map(|c| if c.is_alphanumeric() { c } else { ' ' })
|
||||||
|
.collect::<String>()
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" "),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The display name a slug is stored under: `genres/Science Fiction`. The
|
||||||
|
/// gateway compares both sides folded to lowercase alphanumerics, so
|
||||||
|
/// `science-fiction` and `Science Fiction` are the same genre.
|
||||||
|
fn genre_label(slug: &str) -> String {
|
||||||
|
slug.split(|c: char| !c.is_alphanumeric())
|
||||||
|
.filter(|w| !w.is_empty())
|
||||||
|
.map(|w| {
|
||||||
|
let mut cs = w.chars();
|
||||||
|
match cs.next() {
|
||||||
|
Some(f) => f.to_uppercase().collect::<String>() + cs.as_str(),
|
||||||
|
None => String::new(),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ")
|
||||||
|
}
|
||||||
|
|
||||||
/// The browse markers a card echoes. See [`to_record`].
|
/// The browse markers a card echoes. See [`to_record`].
|
||||||
const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"];
|
const BROWSE_MARKERS: &[&str] = &["popular", "fresh", "trending"];
|
||||||
|
|
||||||
@@ -443,6 +492,35 @@ impl FeederPlugin for OpenLibraryPlugin {
|
|||||||
// A curated row.
|
// A curated row.
|
||||||
if let Some(kind) = browse_kind(query) {
|
if let Some(kind) = browse_kind(query) {
|
||||||
let n = self.discovery_n.min(cap);
|
let n = self.discovery_n.min(cap);
|
||||||
|
// A genre row: `trending/daily` has no subject to cut by, and the
|
||||||
|
// gateway drops any card without `genres/<Name>` (the filter fails
|
||||||
|
// closed), so the row came back empty. Ask Open Library for the
|
||||||
|
// subject instead and stamp the genre the row asked for.
|
||||||
|
let genres = genre_slugs(query);
|
||||||
|
if kind == Browse::Popular && !genres.is_empty() {
|
||||||
|
let mut out: Vec<DiscoveryRecord> = Vec::new();
|
||||||
|
for slug in &genres {
|
||||||
|
let docs = client
|
||||||
|
.by_genre(
|
||||||
|
&genre_subject(slug),
|
||||||
|
(n * SEARCH_OVERFETCH).min(MAX_LIMIT),
|
||||||
|
BROWSE_DEADLINE,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let label = genre_label(slug);
|
||||||
|
for mut r in project(client, docs, query, n, BROWSE_DEADLINE).await {
|
||||||
|
r.fields.insert(format!("genres/{label}"), "true".into());
|
||||||
|
match out.iter_mut().find(|o| o.record_id == r.record_id) {
|
||||||
|
Some(have) => have.fields.extend(
|
||||||
|
r.fields.into_iter().filter(|(k, _)| k.starts_with("genres/")),
|
||||||
|
),
|
||||||
|
None => out.push(r),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.truncate(n);
|
||||||
|
return Ok(out);
|
||||||
|
}
|
||||||
let docs = match kind {
|
let docs = match kind {
|
||||||
Browse::Popular => {
|
Browse::Popular => {
|
||||||
client
|
client
|
||||||
@@ -948,6 +1026,59 @@ mod tests {
|
|||||||
server.verify().await;
|
server.verify().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_genre_row_reads_the_subject_and_stamps_the_genre_it_asked_for() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "subject:\"science fiction\""))
|
||||||
|
.and(query_param("sort", "readinglog"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||||
|
"numFound": 1,
|
||||||
|
"docs": [doc_json("OL66554W", "Nineteen Eighty-Four", Some(14348537))]
|
||||||
|
})))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/trending/daily.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(500))
|
||||||
|
.expect(0)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/works/OL66554W.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(json!({ "key": "/works/OL66554W", "description": "A novel." })))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (p, _dir) = plugin_against(&server).await;
|
||||||
|
let row = q(
|
||||||
|
"",
|
||||||
|
&[("domain", "literature"), ("contentKind", "book"), ("popular", "true"), ("genres", "science-fiction")],
|
||||||
|
);
|
||||||
|
let out = p.handle_query(&row, 20).await.unwrap();
|
||||||
|
assert_eq!(out.len(), 1);
|
||||||
|
assert_eq!(out[0].fields["genres/Science Fiction"], "true");
|
||||||
|
assert_eq!(out[0].fields["popular"], "true");
|
||||||
|
// What the gateway checks before it keeps the card.
|
||||||
|
assert!(meta_feeder_sdk::query_eval::record_matches(&out[0].fields, &row));
|
||||||
|
server.verify().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn genre_slugs_map_to_subjects_and_to_stored_names() {
|
||||||
|
assert_eq!(genre_subject("fantasy"), "fantasy");
|
||||||
|
assert_eq!(genre_subject("science-fiction"), "science fiction");
|
||||||
|
assert_eq!(genre_subject("mystery"), "mystery and detective stories");
|
||||||
|
assert_eq!(genre_subject("a\"b"), "a b", "a quote must not reach the search syntax");
|
||||||
|
assert_eq!(genre_label("science-fiction"), "Science Fiction");
|
||||||
|
assert_eq!(genre_label("fantasy"), "Fantasy");
|
||||||
|
let mut row = q("", &[("popular", "true")]);
|
||||||
|
row.filters.insert("genres".into(), vec!["Fantasy".into(), "fantasy".into(), " ".into()]);
|
||||||
|
assert_eq!(genre_slugs(&row), vec!["fantasy".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn the_fresh_row_sorts_search_by_new() {
|
async fn the_fresh_row_sorts_search_by_new() {
|
||||||
let server = MockServer::start().await;
|
let server = MockServer::start().await;
|
||||||
|
|||||||
@@ -395,6 +395,37 @@ impl OpenLibraryClient {
|
|||||||
Ok(parse_docs(body.get("works")))
|
Ok(parse_docs(body.get("works")))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A genre row (`popular:true genres:<slug>`): the works Open Library readers
|
||||||
|
/// shelve most under one subject, most-shelved first.
|
||||||
|
///
|
||||||
|
/// `trending/daily` carries no subject, so a genre row cannot be cut from it.
|
||||||
|
/// `search.json` filtered by `subject:` and sorted by `readinglog` is the
|
||||||
|
/// same signal (reader shelving) scoped to a genre; as with
|
||||||
|
/// [`Self::trending`], the ordering is used and no count is read or stored.
|
||||||
|
pub async fn by_genre(
|
||||||
|
&self,
|
||||||
|
subject: &str,
|
||||||
|
limit: usize,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
|
let subject = subject.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase();
|
||||||
|
let body = self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:genre:{limit}:{subject}"),
|
||||||
|
BROWSE_TTL_SECS,
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("subject:\"{subject}\"")),
|
||||||
|
("sort", "readinglog".into()),
|
||||||
|
("fields", SEARCH_FIELDS.into()),
|
||||||
|
("limit", limit.to_string()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(parse_docs(body.get("docs")))
|
||||||
|
}
|
||||||
|
|
||||||
/// The `fresh:true` row: recently added works first published this year or
|
/// The `fresh:true` row: recently added works first published this year or
|
||||||
/// last.
|
/// last.
|
||||||
///
|
///
|
||||||
|
|||||||
Reference in New Issue
Block a user