diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs index 2ed0cde..86a1ab2 100644 --- a/crates/meta-feeder-sdk/src/hash.rs +++ b/crates/meta-feeder-sdk/src/hash.rs @@ -409,6 +409,27 @@ fn identity_locator_cid(codec: u64, digest: &[u8]) -> String { format!("b{}", base32_lower_no_padding(&wire)) } +/// Custom multicodec for the **url-locator** family: a CID whose bytes are an +/// `http(s)` URL rather than a hash of the file (METADATA_KEYS.md §2.1), so a +/// record references external bytes without carrying a raw URL field. +/// +/// Copied from the git `meta-feeder-sdk` (`hash.rs`, present since v1.1.0), +/// which this vendored copy predates. Keep the two byte-identical. +pub const URL_LOCATOR_CODEC: u64 = 0x1006; + +/// Encode an `http(s)` URL as a `0x1006` identity-multihash CIDv1. The digest +/// *is* the URL, so minting one costs no I/O. +/// +/// `None` for anything that is not an absolute http(s) URL — a relative value +/// would encode fine and then fail unresolvably at render time. +pub fn compute_url_cid(url: &str) -> Option { + let url = url.trim(); + if !(url.starts_with("http://") || url.starts_with("https://")) { + return None; + } + Some(identity_locator_cid(URL_LOCATOR_CODEC, url.as_bytes())) +} + /// Custom multicodec for a **Usenet posting** identity (`nzb-posting`), /// minted from the article Message-IDs of a release we scanned ourselves. /// @@ -646,6 +667,26 @@ pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { mod tests { use super::*; + /// Pinned against an independent encoder. The tmdb plugin's TS + /// `encodeUrlCid` asserts the same strings, so a drift on either side fails. + /// The long URL exercises the two-byte length varint (> 127 bytes). + #[test] + fn url_cid_matches_pinned_vectors() { + assert_eq!( + compute_url_cid("https://image.tmdb.org/t/p/w500/dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg") + .as_deref(), + Some("bagdcaab7nb2hi4dthixs62lnmftwkltunvsgeltpojts65bpoaxxonjqgaxwi4k2ivhgg2cumq3wy4bvpjuximkrg5bgw2t2mn4ha2jonjygo") + ); + let long = format!("https://image.tmdb.org/t/p/w500/{}.jpg", "a".repeat(100)); + assert_eq!(long.len(), 136); + assert_eq!( + compute_url_cid(&long).as_deref(), + Some("bagdcaaeiafuhi5dqom5c6l3jnvqwozjoorwwiyron5zgol3uf5yc65zvgayc6ylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcltkobtq") + ); + assert!(compute_url_cid("/relative.jpg").is_none()); + assert!(compute_url_cid("ftp://example.com/x.jpg").is_none()); + } + #[test] fn midhash256_matches_ts_fixtures() { assert_eq!( diff --git a/feeder-plugin/card-feeder/src/card.rs b/feeder-plugin/card-feeder/src/card.rs index f9535d3..814a31a 100644 --- a/feeder-plugin/card-feeder/src/card.rs +++ b/feeder-plugin/card-feeder/src/card.rs @@ -17,11 +17,14 @@ use std::collections::BTreeMap; use std::sync::Arc; -use meta_feeder_sdk::hash::compute_card_cid; +use meta_feeder_sdk::hash::{compute_card_cid, compute_url_cid}; use meta_feeder_sdk::types::DiscoveryRecord; use crate::consts::MAX_CARD_SEASONS; -use crate::tmdb_client::{TmdbClient, TmdbKind, TmdbSeasonSummary}; +use crate::tmdb_client::{ + iso639_1_to_3, norm_tokens, normalize_title_name, title_member_key, TmdbClient, TmdbImage, + TmdbKind, TmdbSeasonSummary, +}; /// The metadata source that published a card. One variant per bridge; the /// string form is the CID's source namespace, so **changing it re-mints every @@ -65,14 +68,17 @@ pub(crate) struct Card { pub(crate) title: String, /// Canonical + original + AKA titles. Rides along so the *content* feeders /// can run their resemblance guard against the card instead of re-querying - /// TMDB for it. + /// TMDB for it (phase 2, once the indexer feeder's own anchor resolution is + /// retired). /// - /// Not read here yet — it is consumed by phase 2, which still resolves its - /// own anchor until the indexer feeder's `build_anchor_record` path is - /// retired. Resolving it now means the card is already complete when that - /// switch happens. - #[allow(dead_code)] + /// Read here for one thing: picking `searchTitle` ([`search_aka`]). pub(crate) guard_titles: Arc>, + /// Every name of the work as `(lang3, name)`, filed on the record as the + /// `titles/{lang3}/{name}` key-set (METADATA_KEYS.md). Built by + /// `tmdb_client::title_names`: original under its language, `title` under + /// `eng` when it differs, AKAs by market. Same names as `guard_titles`, with + /// the language each is written in. + pub(crate) names: Vec<(&'static str, String)>, pub(crate) overview: Option, pub(crate) poster_path: Option, /// Editorial genre names (`"Animation"`, `"Drama"`, …) — METADATA_KEYS @@ -98,6 +104,11 @@ pub(crate) struct Card { /// from a mutated series card. #[allow(dead_code)] pub(crate) season_summaries: Arc>, + + /// Poster candidates (TMDB `images.posters`), filed as the + /// `posters/{lang3}/{cid}` key-set by [`poster_member_keys`]. Only the by-id + /// path has them; discovery/search cards leave this empty and file no set. + pub(crate) posters: Arc>, } impl Card { @@ -198,6 +209,23 @@ impl Card { fields.insert("workForm".to_string(), work_form.to_string()); } fields.insert("title".to_string(), self.title.clone()); + // The keyword a consumer should search indexers with, only when `title` + // itself can't be searched (`3%` → `3 percent`). meta-watch's title page + // has no TMDB access of its own and reads this off the card; without it + // its free-text query sends `3%`, which one newznab answers with nothing + // and another floods with every release containing a `3`. + if let Some(aka) = search_aka(&self.title, self.guard_titles.iter()) { + // Normalised like a `titles/*/*` member name: `searchTitle` is always + // one of the record's names, spelled the same way. + fields.insert("searchTitle".to_string(), normalize_title_name(aka)); + } + // Every name the work is known by, as the language-nested key-set. A + // key-set so two peers resolving different AKAs union on key-merge. + for (lang3, name) in &self.names { + if let Some(key) = title_member_key(lang3, name) { + fields.insert(key, "true".to_string()); + } + } // The id bag. if let Some(id) = self.tmdbid { @@ -271,6 +299,15 @@ impl Card { "poster_url".to_string(), tmdb.poster_cdn_url(poster_path), ); + // The alternative posters, as the language-nested key-set (METADATA_KEYS + // §6 `posters/{lang3}/{cid}`). Every member is a `url` locator, so listing + // them costs no fetch. The primary member wraps the exact `poster_url` + // string above: the gateway matches on it and renames the member to the + // content cid it seeds into `poster` — that is how "`poster` is a member" + // holds for a feeder card. + for key in poster_member_keys(tmdb, poster_path, &self.posters) { + fields.insert(key, "true".to_string()); + } for (key, allowed) in query_filters { // `genres` joins `languages` as an echo exclusion, for the same @@ -299,6 +336,117 @@ impl Card { } } +/// Most members a record's `posters/*` key-set may carry, the primary included +/// (METADATA_KEYS.md §6, writer rule 1). +pub(crate) const MAX_POSTER_MEMBERS: usize = 10; + +/// The `posters/{lang3}/{cid}` keys for a card: the primary poster first, then +/// the best-voted alternatives, capped at [`MAX_POSTER_MEMBERS`]. Empty when +/// TMDB returned no image list — without one the primary's language is unknown, +/// and a set is better absent than mislabelled. +/// +/// Ranked by `vote_average`, then `vote_count`, then `file_path`, the last so +/// two peers holding the same payload emit the same keys. The language is that +/// of the text printed on the poster: `zxx` for a textless one (TMDB `null`), +/// `und` for a code outside [`iso639_1_to_3`] or a primary TMDB did not list. +/// +/// CROSS-BINARY CONTRACT: same ranking, cap, CDN size and language rule as the +/// tmdb plugin's `posterMembers` (`metamesh-plugin-tmdb/src/posters.ts`). +pub(crate) fn poster_member_keys( + tmdb: &TmdbClient, + primary_path: &str, + candidates: &[TmdbImage], +) -> Vec { + if candidates.is_empty() { + return Vec::new(); + } + let lang3 = |img: &TmdbImage| -> &'static str { + match img.iso_639_1.as_deref().map(str::trim) { + None | Some("") => "zxx", + Some(code) => iso639_1_to_3(code).unwrap_or("und"), + } + }; + let mut keys: Vec = Vec::new(); + let primary_lang = candidates + .iter() + .find(|img| img.file_path == primary_path) + .map_or("und", lang3); + if let Some(cid) = compute_url_cid(&tmdb.poster_cdn_url(primary_path)) { + keys.push(format!("posters/{primary_lang}/{cid}")); + } + let mut ranked: Vec<&TmdbImage> = candidates + .iter() + .filter(|img| !img.file_path.trim().is_empty() && img.file_path != primary_path) + .collect(); + ranked.sort_by(|a, b| { + b.vote_average + .total_cmp(&a.vote_average) + .then(b.vote_count.cmp(&a.vote_count)) + .then(a.file_path.cmp(&b.file_path)) + }); + for img in ranked { + if keys.len() >= MAX_POSTER_MEMBERS { + break; + } + if let Some(cid) = compute_url_cid(&tmdb.poster_cdn_url(&img.file_path)) { + let key = format!("posters/{}/{cid}", lang3(img)); + if !keys.contains(&key) { + keys.push(key); + } + } + } + keys +} + +/// Punctuation keyword searches fold to a space. Only used to *exclude* these +/// from [`is_search_hostile_symbol`] — a folded character searches fine. +/// +/// CROSS-BINARY CONTRACT: identical to the indexer feeder's +/// `torznab/mod.rs::KEYWORD_PUNCTUATION`, and to meta-watch's +/// `clean_title` + `sanitize_free_text` sets combined. +const KEYWORD_PUNCTUATION: [char; 11] = ['.', '_', ':', ';', ',', '-', '/', '(', ')', '=', '!']; + +/// An ASCII symbol an indexer's search box can neither carry nor safely lose +/// (`%` in `3%`): punctuation outside [`KEYWORD_PUNCTUATION`], apostrophe +/// excepted. See the indexer feeder's `is_search_hostile_symbol` for the +/// measurements. +/// +/// CROSS-BINARY CONTRACT: same rule as the indexer feeder's +/// `torznab/mod.rs::is_search_hostile_symbol` and meta-watch's +/// `catalog::query::has_search_hostile_symbol`. +fn is_search_hostile_symbol(c: char) -> bool { + c.is_ascii_punctuation() && c != '\'' && !KEYWORD_PUNCTUATION.contains(&c) +} + +/// The AKA to search with in place of `title` — the card's `searchTitle`. +/// `None` unless `title` carries a [search-hostile symbol](is_search_hostile_symbol) +/// **and** an AKA is symbol-free and strictly more specific (all of the title's +/// word tokens, plus at least one): `3 percent` for `3%`, never the bare `3`. +/// First qualifying AKA wins; `TmdbAltTitles::all` orders English markets first. +/// +/// CROSS-BINARY CONTRACT: mirrors the indexer feeder's +/// `torznab/mod.rs::search_aka`, which picks the anchored jobs' `q=` the same +/// way. Change both. +pub(crate) fn search_aka<'a>( + title: &str, + akas: impl IntoIterator, +) -> Option<&'a str> { + if !title.chars().any(is_search_hostile_symbol) { + return None; + } + let want = norm_tokens(title); + if want.is_empty() { + return None; + } + akas.into_iter() + .filter(|aka| !aka.chars().any(is_search_hostile_symbol)) + .find(|aka| { + let have = norm_tokens(aka); + have.len() > want.len() && want.iter().all(|w| have.contains(w)) + }) + .map(String::as_str) +} + /// Split a card `record_id` (`"tmdb:tv:95479"`) back into `(source, source_id)` /// — the two halves of the CID preimage. `None` when the id carries no source /// prefix. @@ -320,12 +468,18 @@ mod tests { imdb_id: Some("tt22248376".to_string()), title: "Frieren: Beyond Journey's End".to_string(), guard_titles: Arc::new(vec!["Sousou no Frieren".to_string()]), + names: vec![ + ("jpn", "葬送のフリーレン".to_string()), + ("eng", "Frieren: Beyond Journey's End".to_string()), + ("jpn", "Sousou no Frieren".to_string()), + ], overview: Some("The story follows the elf mage Frieren.".to_string()), poster_path: Some("/dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg".to_string()), genres: vec!["Animation".to_string(), "Sci-Fi & Fantasy".to_string()], year: Some(2023), seasons: 1, season_summaries: Arc::new(Vec::new()), + posters: Arc::new(Vec::new()), } } @@ -394,6 +548,102 @@ mod tests { assert!(no_overview.to_record(&tmdb, &BTreeMap::new()).is_none()); } + /// `3%` (TMDB tv 68467): the card spells the symbol out for searchers, and + /// keeps the real title for display. + #[test] + fn symbol_title_card_carries_a_search_title() { + let tmdb = TmdbClient::new("token".to_string()); + let mut card = tv_card(); + card.title = "3%".to_string(); + card.guard_titles = Arc::new( + ["3%", "3 %", "3", "Three Percent", "3 percent", "3 Pourcent"] + .iter() + .map(|s| s.to_string()) + .collect(), + ); + let rec = card.to_record(&tmdb, &BTreeMap::new()).expect("record"); + assert_eq!(rec.fields["title"], "3%"); + assert_eq!(rec.fields.get("searchTitle").map(String::as_str), Some("3 percent")); + // A searchable title gets no searchTitle — Frieren's apostrophe and + // colon are not hostile. + let rec = tv_card().to_record(&tmdb, &BTreeMap::new()).expect("record"); + assert!(!rec.fields.contains_key("searchTitle")); + } + + #[test] + fn record_files_every_name_as_a_language_nested_key_set() { + let tmdb = TmdbClient::new("token".to_string()); + let rec = tv_card().to_record(&tmdb, &BTreeMap::new()).expect("record"); + for key in [ + "titles/jpn/葬送のフリーレン", + "titles/eng/Frieren: Beyond Journey's End", + "titles/jpn/Sousou no Frieren", + ] { + assert_eq!(rec.fields.get(key).map(String::as_str), Some("true"), "{key}"); + } + // Leaves only: no scalar at the language level. + assert!(!rec.fields.keys().any(|k| k.starts_with("titles/") && k.matches('/').count() == 1)); + } + + /// The METADATA_KEYS.md example, end to end from TMDB's data. + #[test] + fn title_names_match_the_three_percent_example() { + use crate::tmdb_client::{title_names, TmdbAltTitle}; + let aka = |m: &str, t: &str| TmdbAltTitle { title: t.to_string(), iso_3166_1: m.to_string() }; + let names = title_names( + "3%", + Some("3%"), + Some("pt"), + &[aka("US", "3 Percent"), aka("FR", "3 Pourcent"), aka("BR", "3%"), aka("CA", "3 Por Cento ")], + ); + assert_eq!( + names, + vec![ + ("por", "3%".to_string()), + ("eng", "3 Percent".to_string()), + ("fra", "3 Pourcent".to_string()), + ("und", "3 Por Cento".to_string()), + ], + "title == original is filed once; BR `3%` repeats por/3%; CA is multilingual → und" + ); + // A translated title is filed under eng beside the original. + let names = title_names("Naruto", Some("ナルト"), Some("ja"), &[]); + assert_eq!(names, vec![("jpn", "ナルト".to_string()), ("eng", "Naruto".to_string())]); + // Unknown original language → und; TMDB's placeholder is not a name. + assert_eq!(title_names("(untitled)", Some("X"), Some("xx"), &[]), vec![("und", "X".to_string())]); + } + + #[test] + fn title_member_key_normalises_like_two_peers_must() { + assert_eq!(title_member_key("jpn", "Fate/Zero").as_deref(), Some("titles/jpn/Fate\u{2215}Zero")); + assert_eq!(title_member_key("eng", " 3 Percent ").as_deref(), Some("titles/eng/3 Percent")); + assert_eq!(title_member_key("eng", "Pokémon").as_deref(), Some("titles/eng/Pokémon")); + assert_eq!(title_member_key("eng", " "), None); + } + + #[test] + fn search_aka_never_picks_something_weaker() { + let akas = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + assert_eq!(search_aka("M*A*S*H", akas(&["MASH"]).iter()), None); + assert_eq!(search_aka("3%", akas(&["3", "3 %"]).iter()), None); + assert_eq!(search_aka("Law & Order", akas(&["Law and Order"]).iter()), Some("Law and Order")); + assert_eq!(search_aka("Naruto", akas(&["Naruto Shippuden"]).iter()), None); + } + + #[test] + fn tmdb_alt_titles_put_english_markets_first() { + let alt: crate::tmdb_client::TmdbAltTitles = serde_json::from_str( + r#"{"results":[{"iso_3166_1":"FR","title":"3 Pourcent"},{"iso_3166_1":"US","title":"3 percent"},{"title":"no market"}]}"#, + ) + .expect("decode"); + assert_eq!(alt.all(), vec!["3 percent", "3 Pourcent", "no market"]); + assert!(alt.has_markets()); + let legacy: crate::tmdb_client::TmdbAltTitles = + serde_json::from_str(r#"{"results":[{"title":"3 Pourcent"},{"title":"3 percent"}]}"#).expect("decode"); + assert!(!legacy.has_markets(), "a pre-market cache entry must refetch"); + assert!(crate::tmdb_client::TmdbAltTitles::default().has_markets(), "no AKAs is complete"); + } + #[test] fn movie_card_uses_the_movie_content_kind() { let mut card = tv_card(); @@ -431,4 +681,100 @@ mod tests { assert_eq!(rec.fields["genres/Animation"], "true"); assert_eq!(rec.fields["genres/Sci-Fi & Fantasy"], "true"); } + + fn img(path: &str, lang: Option<&str>, avg: f64, count: u32) -> TmdbImage { + TmdbImage { + file_path: path.to_string(), + iso_639_1: lang.map(str::to_string), + vote_average: avg, + vote_count: count, + } + } + + /// METADATA_KEYS §6 `posters/{lang3}/{cid}`: the primary is a member, filed + /// under its own language, as the locator of the exact `poster_url` the + /// gateway seeds — that equality is what lets the gateway rename it. + #[test] + fn record_files_the_primary_poster_as_a_locator_member() { + let tmdb = TmdbClient::new("token".to_string()); + let mut card = tv_card(); + let primary = card.poster_path.clone().expect("fixture poster"); + card.posters = Arc::new(vec![ + img("/alt-fr.jpg", Some("fr"), 5.5, 10), + img(&primary, Some("ja"), 5.0, 3), + img("/textless.jpg", None, 6.0, 1), + ]); + let rec = card.to_record(&tmdb, &BTreeMap::new()).expect("record"); + + let primary_cid = compute_url_cid(&rec.fields["poster_url"]).expect("locator"); + assert_eq!( + rec.fields.get(&format!("posters/jpn/{primary_cid}")).map(String::as_str), + Some("true") + ); + let fr = compute_url_cid(&tmdb.poster_cdn_url("/alt-fr.jpg")).expect("locator"); + let textless = compute_url_cid(&tmdb.poster_cdn_url("/textless.jpg")).expect("locator"); + assert!(rec.fields.contains_key(&format!("posters/fra/{fr}"))); + assert!(rec.fields.contains_key(&format!("posters/zxx/{textless}"))); + let members = rec.fields.keys().filter(|k| k.starts_with("posters/")).count(); + assert_eq!(members, 3, "the primary is listed once, not twice"); + // Leaves only, and never a 639-2/B code. + assert!(!rec.fields.keys().any(|k| k.starts_with("posters/") && k.matches('/').count() != 2)); + assert!(!rec.fields.keys().any(|k| k.starts_with("posters/fre/"))); + } + + #[test] + fn poster_members_rank_by_votes_and_cap_at_ten() { + let tmdb = TmdbClient::new("token".to_string()); + let candidates: Vec = (0..15) + .map(|i| img(&format!("/p{i:02}.jpg"), Some("en"), f64::from(i), 1)) + .collect(); + let keys = poster_member_keys(&tmdb, "/primary.jpg", &candidates); + assert_eq!(keys.len(), MAX_POSTER_MEMBERS); + // Primary first; TMDB didn't list it among its images, so `und`. + let primary = compute_url_cid(&tmdb.poster_cdn_url("/primary.jpg")).expect("locator"); + assert_eq!(keys[0], format!("posters/und/{primary}")); + // Then the best vote_average: p14, p13, … p06 — p05 is cut. + let best = compute_url_cid(&tmdb.poster_cdn_url("/p14.jpg")).expect("locator"); + assert_eq!(keys[1], format!("posters/eng/{best}")); + let cut = compute_url_cid(&tmdb.poster_cdn_url("/p05.jpg")).expect("locator"); + assert!(!keys.contains(&format!("posters/eng/{cut}"))); + // vote_count breaks a vote_average tie. + let tie = poster_member_keys( + &tmdb, + "/primary.jpg", + &[img("/few.jpg", None, 5.0, 1), img("/many.jpg", None, 5.0, 9)], + ); + let many = compute_url_cid(&tmdb.poster_cdn_url("/many.jpg")).expect("locator"); + assert_eq!(tie[1], format!("posters/zxx/{many}")); + // An unmapped language is `und`, never a 2-letter key. + let odd = poster_member_keys(&tmdb, "/primary.jpg", &[img("/x.jpg", Some("xx"), 1.0, 1)]); + assert!(odd[1].starts_with("posters/und/"), "{odd:?}"); + } + + /// No image list (a discovery hit, a pre-append cache entry): no set at all, + /// rather than a primary filed under a guessed `und`. + #[test] + fn card_without_an_image_list_files_no_poster_set() { + let tmdb = TmdbClient::new("token".to_string()); + let rec = tv_card().to_record(&tmdb, &BTreeMap::new()).expect("record"); + assert!(rec.fields.contains_key("poster_url"), "poster itself is unaffected"); + assert!(!rec.fields.keys().any(|k| k.starts_with("posters/"))); + } + + #[test] + fn tmdb_images_append_decodes() { + let d: crate::tmdb_client::TmdbTvDetails = serde_json::from_str( + r#"{"name":"X","images":{"posters":[{"file_path":"/a.jpg","iso_639_1":null,"vote_average":5.3,"vote_count":4},{"file_path":"/b.jpg","iso_639_1":"fr","vote_average":0}]}}"#, + ) + .expect("decode"); + let posters = d.images.expect("images").posters; + assert_eq!(posters.len(), 2); + assert_eq!(posters[0].iso_639_1, None); + assert_eq!(posters[1].iso_639_1.as_deref(), Some("fr")); + assert_eq!(posters[1].vote_count, 0); + // A pre-append cache entry still decodes, with no image list. + let old: crate::tmdb_client::TmdbTvDetails = + serde_json::from_str(r#"{"name":"X"}"#).expect("decode"); + assert!(old.images.is_none()); + } } diff --git a/feeder-plugin/card-feeder/src/discovery.rs b/feeder-plugin/card-feeder/src/discovery.rs index 2fab969..33b1e6e 100644 --- a/feeder-plugin/card-feeder/src/discovery.rs +++ b/feeder-plugin/card-feeder/src/discovery.rs @@ -60,7 +60,7 @@ use tracing::debug; use crate::card::{Card, CardSource}; use crate::consts::{DISCOVERY_EXCLUDED_KEYWORDS, DISCOVERY_MAX_PAGES, DISCOVERY_MIN_VOTES}; use crate::resolve::{dedup_titles, Resolver}; -use crate::tmdb_client::{TmdbHit, TmdbKind}; +use crate::tmdb_client::{title_names, TmdbHit, TmdbKind}; /// TMDB keyword id for "anime". `/discover` is the only catalog endpoint that /// takes a keyword filter, which is why `anime:true` switches endpoints rather @@ -300,6 +300,14 @@ fn card_from_hit(hit: &TmdbHit, kind: TmdbKind, genres: &HashMap) - imdb_id: None, title: hit.title.clone(), guard_titles: Arc::new(dedup_titles(titles)), + // A list hit carries no AKAs, so this is title + original only; the + // click-time `card_by_id` card adds the AKAs under the same CID. + names: title_names( + &hit.title, + hit.original_title.as_deref(), + hit.original_language.as_deref(), + &hit.akas, + ), overview: hit.overview.clone().filter(|s| !s.trim().is_empty()), poster_path: hit.poster_path.clone().filter(|s| !s.trim().is_empty()), // A list hit carries bare genre ids; the table names them. @@ -309,6 +317,10 @@ fn card_from_hit(hit: &TmdbHit, kind: TmdbKind, genres: &HashMap) - // omitted from the record (`Card::to_record` skips it at `seasons == 0`). seasons: 0, season_summaries: Arc::new(Vec::new()), + // A list response carries no `images` append, so a discovery card files + // no `posters/*` set; the click-time `card_by_id` card adds it under the + // same CID. + posters: Arc::new(Vec::new()), } } @@ -592,6 +604,7 @@ mod tests { fn hit() -> TmdbHit { TmdbHit { + akas: Vec::new(), tmdbid: 95479, title: "Frieren: Beyond Journey's End".to_string(), original_title: Some("葬送のフリーレン".to_string()), @@ -601,6 +614,7 @@ mod tests { poster_path: Some("/poster.jpg".to_string()), genre_ids: vec![16], alt_titles: Vec::new(), + posters: Vec::new(), } } diff --git a/feeder-plugin/card-feeder/src/resolve.rs b/feeder-plugin/card-feeder/src/resolve.rs index feea540..b38915e 100644 --- a/feeder-plugin/card-feeder/src/resolve.rs +++ b/feeder-plugin/card-feeder/src/resolve.rs @@ -22,7 +22,8 @@ use crate::consts::{ }; use crate::tmdb_budget::{Lease, TmdbBudget}; use crate::tmdb_client::{ - principal_top_n, TmdbCall, TmdbClient, TmdbExternalIds, TmdbHit, TmdbKind, TmdbTvDetails, + principal_top_n, title_names, TmdbAltTitles, TmdbCall, TmdbClient, TmdbExternalIds, TmdbHit, + TmdbKind, TmdbTvDetails, }; /// TMDB resolution with a persistent cache in front and a token budget behind. @@ -149,7 +150,11 @@ impl Resolver { if let Ok(Some(json)) = self.cache.get_tmdb_moviedetails(&key) { if json != "null" { if let Ok(hit) = serde_json::from_str::(&json) { - return Some(hit); + // Self-heal: AKAs cached without their markets can't be + // filed under `titles/{lang3}/{name}` — refetch once. + if hit.alt_titles.is_empty() || !hit.akas.is_empty() { + return Some(hit); + } } } } @@ -265,7 +270,18 @@ impl Resolver { .as_deref() .and_then(|d| d.get(0..4)) .and_then(|y| y.parse::().ok()); + let names = title_names( + &details.name, + details.original_name.as_deref(), + details.original_language.as_deref(), + &details + .alternative_titles + .as_ref() + .map(TmdbAltTitles::entries) + .unwrap_or_default(), + ); Some(Card { + names, source: CardSource::Tmdb, kind, source_id: format!("tv:{tmdbid}"), @@ -280,6 +296,13 @@ impl Resolver { year, seasons: Card::clamp_seasons(details.number_of_seasons), season_summaries: Arc::new(details.seasons.clone()), + posters: Arc::new( + details + .images + .as_ref() + .map(|i| i.posters.clone()) + .unwrap_or_default(), + ), }) } TmdbKind::Movie => { @@ -298,7 +321,14 @@ impl Resolver { titles.push(o.clone()); } titles.extend(hit.alt_titles.iter().cloned()); + let names = title_names( + &hit.title, + hit.original_title.as_deref(), + hit.original_language.as_deref(), + &hit.akas, + ); Some(Card { + names, source: CardSource::Tmdb, kind, source_id: format!("movie:{tmdbid}"), @@ -313,6 +343,7 @@ impl Resolver { year: hit.year, seasons: 0, season_summaries: Arc::new(Vec::new()), + posters: Arc::new(hit.posters), }) } } diff --git a/feeder-plugin/card-feeder/src/tmdb_client.rs b/feeder-plugin/card-feeder/src/tmdb_client.rs index f0b67ba..d759d38 100644 --- a/feeder-plugin/card-feeder/src/tmdb_client.rs +++ b/feeder-plugin/card-feeder/src/tmdb_client.rs @@ -190,7 +190,7 @@ impl TmdbClient { /// any failure returns `None` so validation degrades to "accept". pub(crate) async fn tv_details(&self, tmdbid: u64) -> TmdbCall { let url = format!( - "{}/tv/{}?append_to_response=alternative_titles", + "{}/tv/{}?append_to_response=alternative_titles,images", self.api_base.trim_end_matches('/'), tmdbid, ); @@ -253,7 +253,7 @@ impl TmdbClient { /// [`tv_details`]. pub(crate) async fn movie_details(&self, tmdbid: u64) -> TmdbCall { let url = format!( - "{}/movie/{}?append_to_response=alternative_titles", + "{}/movie/{}?append_to_response=alternative_titles,images", self.api_base.trim_end_matches('/'), tmdbid ); @@ -457,8 +457,8 @@ pub(crate) struct TmdbSearchItem { #[serde(default)] pub(crate) genre_ids: Vec, /// ISO 639-1 (2-letter) language the title was originally produced in. - /// Mapped to `lang3` and used to file `original_title` under - /// `titles/{lang3}` (METADATA_KEYS.md §3). + /// Mapped to `lang3` and used to file `original_title` as a + /// `titles/{lang3}/{name}` member (METADATA_KEYS.md §3). #[serde(default)] pub(crate) original_language: Option, } @@ -493,6 +493,8 @@ impl TmdbSearchItem { genre_ids: self.genre_ids, // search-list items carry no AKAs (a details/append fetch does) alt_titles: Vec::new(), + akas: Vec::new(), + posters: Vec::new(), } } } @@ -500,8 +502,8 @@ impl TmdbSearchItem { /// Map an ISO 639-1 (2-letter) code to its ISO 639-3 (`lang3`) equivalent, /// matching the store's convention (`eng`, `jpn`, `fra`; the 639-2/T variant /// where B/T differ, e.g. `deu` not `ger`). Covers the languages TMDB -/// commonly returns; unknown codes return `None` so the caller skips the -/// `titles/{lang3}` write rather than persisting a non-`lang3` key. +/// commonly returns; unknown codes return `None` so the caller files the name under `und` +/// rather than persisting a non-`lang3` key. pub(crate) fn iso639_1_to_3(code: &str) -> Option<&'static str> { Some(match code.to_ascii_lowercase().as_str() { "en" => "eng", @@ -541,6 +543,113 @@ pub(crate) fn iso639_1_to_3(code: &str) -> Option<&'static str> { }) } +/// The language TMDB's `name`/`title` is written in. Every request goes out +/// without a `language` param, so TMDB answers in its default, en-US. +pub(crate) const METADATA_LANG3: &str = "eng"; + +/// A title as a `titles/{lang3}/{name}` member name (METADATA_KEYS.md): trimmed, +/// internal whitespace collapsed to one space, case/diacritics/`%` verbatim. +/// The `/` → `∕` mapping is the key's job ([`title_member_key`]), not the name's. +pub(crate) fn normalize_title_name(name: &str) -> String { + name.split_whitespace().collect::>().join(" ") +} + +/// The `titles/{lang3}/{name}` key-set member for `name`, or `None` when the +/// name normalises to empty. `/` is the key-set separator, so a `/` inside a +/// name (`Fate/Zero`) is written as U+2215 `∕` (METADATA_KEYS.md). +/// +/// CROSS-BINARY CONTRACT: the indexer feeder's `tmdb.rs::title_member_key` +/// writes the same shape; two peers must produce the same key for one name. +pub(crate) fn title_member_key(lang3: &str, name: &str) -> Option { + let name = normalize_title_name(name); + if name.is_empty() { + return None; + } + Some(format!("titles/{lang3}/{}", name.replace('/', "\u{2215}"))) +} + +/// The language a TMDB AKA market (`iso_3166_1`) clearly implies, or `und` for +/// a multilingual or unknown market (METADATA_KEYS.md `titles/{lang3}/{name}`: +/// map what is unambiguous, never guess). `CA`, `CH`, `BE`, `IN`, `HK`, `SG`, +/// empty, … are `und` on purpose. +pub(crate) fn market_lang3(market: &str) -> &'static str { + match market.trim().to_ascii_uppercase().as_str() { + "US" | "GB" | "AU" | "NZ" | "IE" => "eng", + "FR" => "fra", + "DE" | "AT" => "deu", + "ES" | "MX" | "AR" | "CO" | "CL" | "PE" | "VE" | "UY" => "spa", + "BR" | "PT" => "por", + "IT" => "ita", + "JP" => "jpn", + "KR" => "kor", + "CN" | "TW" => "zho", + "RU" => "rus", + "UA" => "ukr", + "PL" => "pol", + "NL" => "nld", + "SE" => "swe", + "NO" => "nor", + "DK" => "dan", + "FI" => "fin", + "TR" => "tur", + "GR" => "ell", + "HU" => "hun", + "CZ" => "ces", + "SK" => "slk", + "RO" => "ron", + "BG" => "bul", + "RS" => "srp", + "HR" => "hrv", + "IL" => "heb", + "IR" => "fas", + "TH" => "tha", + "VN" => "vie", + "ID" => "ind", + _ => "und", + } +} + +/// Every name of a work as `(lang3, name)` — what the card files under +/// `titles/{lang3}/{name}`: +/// +/// - `original` under its `original_language` (`und` when unmapped/absent); +/// - `title` under [`METADATA_LANG3`], **unless it equals the original** — TMDB +/// falls back to the original name when it has no en-US translation, so `3%` +/// is filed once, as `titles/por/3%`, not also as an English name; +/// - every AKA under the language its market implies ([`market_lang3`]). +/// +/// Names are normalised ([`normalize_title_name`]); blanks, TMDB's +/// `(untitled)` placeholder and exact `(lang3, name)` repeats are dropped. +pub(crate) fn title_names( + title: &str, + original: Option<&str>, + original_language: Option<&str>, + akas: &[TmdbAltTitle], +) -> Vec<(&'static str, String)> { + let mut out: Vec<(&'static str, String)> = Vec::new(); + let mut push = |lang3: &'static str, name: &str| { + let name = normalize_title_name(name); + if !name.is_empty() && !out.iter().any(|(l, n)| *l == lang3 && *n == name) { + out.push((lang3, name)); + } + }; + let original = original.map(normalize_title_name).filter(|o| !o.is_empty()); + if let Some(o) = &original { + let lang3 = original_language + .and_then(|l| iso639_1_to_3(l.trim())) + .unwrap_or("und"); + push(lang3, o); + } + let title = normalize_title_name(title); + if title != "(untitled)" && original.as_deref() != Some(title.as_str()) { + push(METADATA_LANG3, &title); + } + for aka in akas { + push(market_lang3(&aka.iso_3166_1), &aka.title); + } + out +} + // `Clone` is required so the single-flight `Shared` future (whose `Output` // must be `Clone`) can hand the same hit to every coalesced caller; serde so // hits persist in the redb TMDB-search cache. @@ -566,17 +675,53 @@ pub(crate) struct TmdbHit { /// Empty for hits built from a search list (which carries no AKAs). #[serde(default)] pub(crate) alt_titles: Vec, + /// The same AKAs **with their market**, which the card needs to file each + /// under `titles/{lang3}/{name}` ([`title_names`]). Empty on a movie-details + /// cache entry written before it existed — `Resolver::movie_hit` refetches + /// those once. + #[serde(default)] + pub(crate) akas: Vec, + /// Poster candidates from the details `images` append, filed by the card as + /// `posters/{lang3}/{cid}` (METADATA_KEYS.md §6). Empty for list hits and for + /// a movie-details cache entry written before it existed — deliberately NOT a + /// self-heal trigger in `Resolver::movie_hit`: no backfill, no TMDB budget. + #[serde(default)] + pub(crate) posters: Vec, } impl TmdbHit { /// TMDB `original_language` (ISO 639-1) mapped to the store's `lang3` /// (ISO 639-3), or `None` when the language is outside the common set or - /// absent. Used to file `original_title` under `titles/{lang3}` (§3). + /// absent. Used to file `original_title` under `titles/{lang3}/{name}` (§3). pub(crate) fn original_lang3(&self) -> Option<&'static str> { iso639_1_to_3(self.original_language.as_deref()?.trim()) } } +/// One poster from a details payload's `images` append — a candidate for the +/// card's `posters/{lang3}/{cid}` key-set ([`crate::card::poster_member_keys`]). +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct TmdbImage { + #[serde(default)] + pub(crate) file_path: String, + /// Language of the text printed on the poster (ISO 639-1); `None` = textless. + #[serde(default)] + pub(crate) iso_639_1: Option, + #[serde(default)] + pub(crate) vote_average: f64, + #[serde(default)] + pub(crate) vote_count: u32, +} + +/// The `images` object `?append_to_response=images` adds to `GET /3/{tv,movie}/{id}`. +/// Only `posters` is decoded. No `language` param is sent, so TMDB returns the +/// posters of every language. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub(crate) struct TmdbImages { + #[serde(default)] + pub(crate) posters: Vec, +} + /// Authoritative TV structure from `GET /3/tv/{id}`, used to bounds-check /// a title-parsed season/episode. Only the two structural fields are /// decoded; everything else in the (large) details payload is ignored. @@ -617,6 +762,10 @@ pub(crate) struct TmdbTvDetails { /// fetch self-heals with one refetch (same pattern as the display fields). #[serde(default)] pub(crate) alternative_titles: Option, + /// Poster candidates from `?append_to_response=images`. `None` on a cache + /// entry written before the append — NOT a self-heal trigger (no backfill). + #[serde(default)] + pub(crate) images: Option, } impl TmdbTvDetails { @@ -630,8 +779,15 @@ impl TmdbTvDetails { /// True when the AKA field was populated by the current (append-enabled) /// decoder. A `false` on a cache hit means a pre-AKA entry → the cached /// fetch refetches once to upgrade it (mirrors [`has_display`]). + /// + /// Also `false` for an AKA list decoded before `iso_3166_1` was: the + /// search-term pick (`card::search_aka`) prefers English-market AKAs, and a + /// market-less list would silently keep TMDB's order (`3 Pourcent` over + /// `3 percent`) forever. One refetch per such entry upgrades it. pub(crate) fn has_akas(&self) -> bool { - self.alternative_titles.is_some() + self.alternative_titles + .as_ref() + .is_some_and(TmdbAltTitles::has_markets) } /// The show's canonical + original + AKA titles, for the id-based-job @@ -672,6 +828,16 @@ impl TmdbTvDetails { .as_ref() .map(TmdbAltTitles::all) .unwrap_or_default(), + akas: self + .alternative_titles + .as_ref() + .map(TmdbAltTitles::entries) + .unwrap_or_default(), + posters: self + .images + .as_ref() + .map(|i| i.posters.clone()) + .unwrap_or_default(), }) } } @@ -699,17 +865,42 @@ pub(crate) struct TmdbAltTitles { pub(crate) struct TmdbAltTitle { #[serde(default)] pub(crate) title: String, + /// The market the AKA is used in (`US`, `FR`, …). Empty on a cache entry + /// written before it was decoded — ordering then falls back to TMDB's. + #[serde(default)] + pub(crate) iso_3166_1: String, } impl TmdbAltTitles { + /// Was this list decoded with markets? TMDB stamps `iso_3166_1` on every + /// AKA, so a non-empty list where none carries one is a pre-market cache + /// entry. An empty list is complete ("fetched, no AKAs"). + pub(crate) fn has_markets(&self) -> bool { + let mut titles = self.results.iter().chain(self.titles.iter()).peekable(); + titles.peek().is_none() || titles.any(|t| !t.iso_3166_1.is_empty()) + } + /// The non-empty AKA titles across both the TV (`results`) and movie - /// (`titles`) shapes. + /// (`titles`) shapes, **English-market (`US`, `GB`) titles first**, TMDB's + /// order otherwise. The relevance guard only asks "any of these?", so the + /// order is free for it; it matters to the search-term AKA pick + /// (`card::search_aka`), where `3 percent` (US) must beat `3 Pourcent` (FR). pub(crate) fn all(&self) -> Vec { - self.results - .iter() - .chain(self.titles.iter()) - .map(|t| t.title.trim().to_string()) - .filter(|t| !t.is_empty()) + self.entries().into_iter().map(|t| t.title).collect() + } + + /// [`Self::all`] with each AKA's market kept — same order, same filtering + /// (titles trimmed, blanks dropped). + pub(crate) fn entries(&self) -> Vec { + let mut titles: Vec<&TmdbAltTitle> = self.results.iter().chain(self.titles.iter()).collect(); + titles.sort_by_key(|t| !matches!(t.iso_3166_1.as_str(), "US" | "GB")); + titles + .into_iter() + .map(|t| TmdbAltTitle { + title: t.title.trim().to_string(), + iso_3166_1: t.iso_3166_1.trim().to_string(), + }) + .filter(|t| !t.title.is_empty()) .collect() } } @@ -748,6 +939,9 @@ pub(crate) struct TmdbMovieDetails { pub(crate) poster_path: Option, #[serde(default)] pub(crate) alternative_titles: Option, + /// See [`TmdbTvDetails::images`]. + #[serde(default)] + pub(crate) images: Option, } impl TmdbMovieDetails { @@ -762,6 +956,12 @@ impl TmdbMovieDetails { .as_ref() .map(TmdbAltTitles::all) .unwrap_or_default(); + let akas = self + .alternative_titles + .as_ref() + .map(TmdbAltTitles::entries) + .unwrap_or_default(); + let posters = self.images.map(|i| i.posters).unwrap_or_default(); TmdbHit { tmdbid: self.id, title: self.title.unwrap_or_else(|| "(untitled)".to_string()), @@ -772,6 +972,8 @@ impl TmdbMovieDetails { poster_path: self.poster_path, genre_ids: Vec::new(), alt_titles, + akas, + posters, } } } diff --git a/feeder-plugin/card-feeder/tests/contract.rs b/feeder-plugin/card-feeder/tests/contract.rs index bfc9a78..06355d3 100644 --- a/feeder-plugin/card-feeder/tests/contract.rs +++ b/feeder-plugin/card-feeder/tests/contract.rs @@ -64,7 +64,7 @@ fn tv_details_json() -> serde_json::Value { "poster_path": "/poster.jpg", "number_of_seasons": 1, "seasons": [{"season_number": 1, "episode_count": 28}], - "alternative_titles": { "results": [{"title": "Sousou no Frieren"}] } + "alternative_titles": { "results": [{"iso_3166_1": "JP", "title": "Sousou no Frieren"}] } }) } @@ -148,6 +148,15 @@ async fn query_returns_a_card_with_both_axes_and_the_id_bag() { assert_eq!(f["fileType"], "card"); assert_eq!(f["contentKind"], "series"); assert_eq!(f["title"], "Frieren: Beyond Journey's End"); + // Every name as the language-nested key-set: original (ja), the en-US + // title, and the JP-market AKA. + for key in [ + "titles/jpn/\u{845}\u{9001}\u{306e}\u{30d5}\u{30ea}\u{30fc}\u{30ec}\u{30f3}", + "titles/eng/Frieren: Beyond Journey's End", + "titles/jpn/Sousou no Frieren", + ] { + assert_eq!(f.get(key).map(String::as_str), Some("true"), "{key}"); + } // The id bag — phase 2 selects from this rather than assuming `tmdbid`. assert_eq!(f["tmdbid"], "95479"); assert_eq!(f["tvdbid"], "367189");