From 631beb11413410ccf8d3b4247d21917fd3c1ab86 Mon Sep 17 00:00:00 2001 From: Worph Date: Sat, 1 Aug 2026 20:22:06 +0200 Subject: [PATCH] genre management --- crates/meta-feeder-sdk/src/query_eval.rs | 58 ++ feeder-plugin/card-feeder/src/card.rs | 91 ++- feeder-plugin/card-feeder/src/consts.rs | 73 ++ feeder-plugin/card-feeder/src/discovery.rs | 708 +++++++++++++++++++ feeder-plugin/card-feeder/src/lib.rs | 1 + feeder-plugin/card-feeder/src/resolve.rs | 114 ++- feeder-plugin/card-feeder/src/tmdb.rs | 53 +- feeder-plugin/card-feeder/src/tmdb_client.rs | 43 ++ 8 files changed, 1125 insertions(+), 16 deletions(-) create mode 100644 feeder-plugin/card-feeder/src/discovery.rs diff --git a/crates/meta-feeder-sdk/src/query_eval.rs b/crates/meta-feeder-sdk/src/query_eval.rs index 523187d..c0d5f81 100644 --- a/crates/meta-feeder-sdk/src/query_eval.rs +++ b/crates/meta-feeder-sdk/src/query_eval.rs @@ -27,6 +27,16 @@ pub fn record_matches(fields: &BTreeMap, query: &GatewayQuery) - } continue; } + // `genres` is a key-set too (`genres/ = "true"`, METADATA_KEYS + // §`genres/{name}`) on everything the card tier writes. Without this arm + // a `genres:` filter looks up a flat `genres` field, finds nothing on a + // card, and drops it — so a genre-scoped row would return empty. + if key == "genres" { + if !genres_filter_matches(fields, allowed) { + return false; + } + continue; + } let Some(field_value) = fields.get(key) else { return false; }; @@ -155,6 +165,54 @@ fn record_has_concrete_language(fields: &BTreeMap) -> bool { /// Key-set members are written as the literal `"true"`; tolerate a couple of /// obvious truthy spellings and treat anything else as absent. +/// Fold a genre label to its comparison form: lowercase, alphanumerics only. +/// +/// Genre display names carry spaces and ampersands (`"Action & Adventure"`, +/// `"Sci-Fi & Fantasy"`, `"Science Fiction"`), and **space is the query DSL's +/// token separator** — so a filter can never spell a genre the way the record +/// stores it. Callers pass a slug (`genres:action-adventure`); folding both +/// sides makes them one value without forcing the stored name to be +/// machine-shaped. +fn genre_fold(s: &str) -> String { + s.chars() + .filter(|c| c.is_alphanumeric()) + .flat_map(|c| c.to_lowercase()) + .collect() +} + +/// True iff the record carries one of the requested genres, in **either** +/// encoding: a truthy `genres/` key-set member (the card tier's shape) or +/// the legacy flat comma-joined `genres` field. OR across the requested genres, +/// both sides folded — see [`genre_fold`]. +/// +/// **Fails closed, unlike [`languages_filter_matches`].** A language filter is a +/// viewer *preference* narrowing a set, so an unknown-language release is kept. +/// A genre filter *defines what a row is*: a record with no genre information is +/// not known to be Action, so putting it in an Action row states something +/// untrue. Upstream has already filtered by genre, so this is a re-validation. +fn genres_filter_matches(fields: &BTreeMap, allowed: &[String]) -> bool { + let wanted: Vec = allowed.iter().map(|g| genre_fold(g)).collect(); + if wanted.iter().all(|g| g.is_empty()) { + return true; // a filter with no usable value constrains nothing + } + for (key, value) in fields { + let Some(name) = key.strip_prefix("genres/") else { + continue; + }; + if is_truthy_member(value) && wanted.contains(&genre_fold(name)) { + return true; + } + } + if let Some(flat) = fields.get("genres") { + for member in flat.split(|c| c == ',' || c == '|') { + if wanted.contains(&genre_fold(member)) { + return true; + } + } + } + false +} + fn is_truthy_member(v: &str) -> bool { matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes") } diff --git a/feeder-plugin/card-feeder/src/card.rs b/feeder-plugin/card-feeder/src/card.rs index 0286ed0..b819040 100644 --- a/feeder-plugin/card-feeder/src/card.rs +++ b/feeder-plugin/card-feeder/src/card.rs @@ -75,6 +75,17 @@ pub(crate) struct Card { pub(crate) guard_titles: Arc>, pub(crate) overview: Option, pub(crate) poster_path: Option, + /// Editorial genre names (`"Animation"`, `"Drama"`, …) — METADATA_KEYS + /// §`genres`. The *work's* categorisation, distinct from `categories/*`, + /// which is the Newznab/Prowlarr distribution taxonomy a release is filed + /// under and which a card (having no release) never carries. + /// + /// Free on both resolution paths: a details fetch returns named genre + /// objects, and a list hit's bare `genre_ids` are mapped through + /// `Resolver::genre_names`. May be empty — TMDB genre coverage is good but + /// not universal, and consumers must treat "no genres" as unknown rather + /// than as "none". + pub(crate) genres: Vec, pub(crate) year: Option, // -- structure (TV only) ------------------------------------------------ @@ -123,6 +134,27 @@ impl Card { n.clamp(1, MAX_CARD_SEASONS) } + /// The card's synopsis, trimmed and non-empty. + pub(crate) fn overview_text(&self) -> Option<&str> { + self.overview.as_deref().map(str::trim).filter(|s| !s.is_empty()) + } + + /// The card's poster path, non-empty. + pub(crate) fn poster(&self) -> Option<&str> { + self.poster_path.as_deref().filter(|s| !s.is_empty()) + } + + /// Would this card actually render? Exactly [`Card::to_record`]'s emit + /// condition, hoisted so a caller can predict the drop *before* projecting. + /// + /// [`crate::discovery`] needs this: it walks catalog pages until it has N + /// cards, and counting raw TMDB hits would overcount — a hit with no poster + /// or no overview is silently declined at projection time, leaving a short + /// row. The two must not drift, hence one predicate rather than two. + pub(crate) fn is_displayable(&self) -> bool { + self.overview_text().is_some() && self.poster().is_some() + } + /// Project to the wire record. /// /// Returns `None` when the card lacks a poster or a description — it would @@ -139,8 +171,10 @@ impl Card { tmdb: &TmdbClient, query_filters: &BTreeMap>, ) -> Option { - let overview = self.overview.as_deref().map(str::trim).filter(|s| !s.is_empty())?; - let poster_path = self.poster_path.as_deref().filter(|s| !s.is_empty())?; + // Same two conditions as `Card::is_displayable`, which the discovery page + // walk uses to predict this drop — keep them in one place. + let overview = self.overview_text()?; + let poster_path = self.poster()?; let mut fields: BTreeMap = BTreeMap::new(); // Type axes. `card` on the fileType axis and the work kind on the @@ -171,6 +205,24 @@ impl Card { // `overview`). fields.insert("description/eng".to_string(), overview.to_string()); + // Genres as a KEY-SET (`genres/ = "true"`), not the legacy + // comma-joined `genres` value the tmdb/jellyfin plugins still write. + // + // METADATA_KEYS §14.12 is explicit that `csv-set` is the shape being + // migrated *away* from and that "new writers must not introduce more + // csv-set fields — reach for key-set, even if the surrounding family is + // still on the legacy shape". This is a new writer, and the reason bites + // here specifically: a card is the one record type multiple peers derive + // independently at the same CID (§9.4), so two peers resolving different + // genre sets must union by key-merge rather than diverge into two + // comma-joined strings that need string-diffing to reconcile. + for g in &self.genres { + let g = g.trim(); + if !g.is_empty() { + fields.insert(format!("genres/{g}"), "true".to_string()); + } + } + // The bare-CID key-set member, exactly as every other feeder stamps on // its search records (`indexer-feeder`'s `torznab/xml.rs` does the same // with the release's infohash). @@ -206,7 +258,19 @@ impl Card { ); for (key, allowed) in query_filters { - if key == "languages" || fields.contains_key(key) || allowed.is_empty() { + // `genres` joins `languages` as an echo exclusion, for the same + // reason: it is stored as a key-set, so the flat field this loop + // would write is not the shape a reader looks for. Worse, the filter + // value is a *slug* (`action-adventure` — the query DSL can't spell + // "Action & Adventure", space being its token separator), so echoing + // it would persist a slug masquerading as a genre name alongside the + // real `genres/` members. Both re-validating tiers read the + // key-set, so nothing needs the echo. + if key == "languages" + || key == "genres" + || fields.contains_key(key) + || allowed.is_empty() + { continue; } fields.insert(key.clone(), allowed.join(",")); @@ -243,6 +307,7 @@ mod tests { guard_titles: Arc::new(vec!["Sousou no Frieren".to_string()]), overview: Some("The story follows the elf mage Frieren.".to_string()), poster_path: Some("/dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg".to_string()), + genres: vec!["Animation".to_string(), "Sci-Fi & Fantasy".to_string()], year: Some(2023), seasons: 1, season_summaries: Arc::new(Vec::new()), @@ -326,17 +391,29 @@ mod tests { } /// Query filters are echoed so the record survives `record_matches` on the - /// way out, but `languages` is deliberately skipped (it fails open). + /// way out — except the two key-set fields, which a reader resolves from + /// their `/` members rather than from a flat value. #[test] - fn query_filters_are_echoed_except_languages() { + fn query_filters_are_echoed_except_the_key_set_fields() { let tmdb = TmdbClient::new("token".to_string()); let mut filters = BTreeMap::new(); filters.insert("fileType".to_string(), vec!["card".to_string()]); - filters.insert("genres".to_string(), vec!["fantasy".to_string()]); + filters.insert("popular".to_string(), vec!["true".to_string()]); + filters.insert("genres".to_string(), vec!["action-adventure".to_string()]); filters.insert("languages".to_string(), vec!["jpn".to_string()]); let rec = tv_card().to_record(&tmdb, &filters).expect("record"); - assert_eq!(rec.fields["genres"], "fantasy"); + + assert_eq!(rec.fields["popular"], "true", "ordinary filters echo"); assert_eq!(rec.fields["fileType"], "card", "own value wins the echo"); assert!(!rec.fields.contains_key("languages")); + // The genre filter carries a *slug* the DSL can spell; echoing it would + // persist "action-adventure" as if it were a genre name, beside the real + // `genres/Action & Adventure` members. + assert!( + !rec.fields.contains_key("genres"), + "the genre slug must not be echoed as a flat field" + ); + assert_eq!(rec.fields["genres/Animation"], "true"); + assert_eq!(rec.fields["genres/Sci-Fi & Fantasy"], "true"); } } diff --git a/feeder-plugin/card-feeder/src/consts.rs b/feeder-plugin/card-feeder/src/consts.rs index c2615e5..6881abe 100644 --- a/feeder-plugin/card-feeder/src/consts.rs +++ b/feeder-plugin/card-feeder/src/consts.rs @@ -30,6 +30,70 @@ pub(crate) const TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS: u64 = 5; /// Maximum time to wait for a TMDB budget permit before giving up (seconds). pub(crate) const TMDB_WAIT_DEADLINE_SECS: u64 = 3000; +/// Budget-permit deadline for a **catalog page** ([`crate::discovery`]), +/// deliberately far shorter than [`TMDB_WAIT_DEADLINE_SECS`]. +/// +/// That 3000 s ceiling is sized for a query someone is *waiting on* — a user +/// typed a title and wants that specific answer, so out-waiting a 429 pause +/// beats returning nothing. A browse row is the opposite: nobody asked for +/// *this* row in particular, it refreshes on a timer, and its client keeps the +/// previously-filled row on an empty pass (meta-watch's sticky per-row merge). +/// Inheriting 3000 s would let a single 429 pin a row's search stream open for +/// most of an hour to eventually produce what a 5 s bail produces now. +pub(crate) const TMDB_DISCOVERY_WAIT_DEADLINE_SECS: u64 = 5; + +/// Hard ceiling on catalog pages walked for one discovery row, whatever the +/// requested card count. Bounds a misconfigured `card_discovery_n` (or a client +/// asking for 500) to a predictable per-row TMDB cost. +pub(crate) const DISCOVERY_MAX_PAGES: u32 = 5; + +/// TMDB keyword ids excluded from an `anime:true` discovery row, as +/// `without_keywords`: `hentai`, `softcore`, `erotica`, `pornography`, +/// `adult animation`. +/// +/// **Why this is needed at all.** `include_adult=false` does *not* cover it: +/// TMDB's `adult` boolean is reserved for actual pornography, and every one of +/// the titles this excludes comes back `adult: false`. Measured on the live +/// endpoint, the flag removed 84 of 4465 anime series and none of the offenders. +/// +/// **Why they surface in the first place.** `sort_by=popularity.desc` sorts on +/// TMDB's *traffic* metric (page views, API hits, list adds), not on audience +/// size or rating. Adult titles attract disproportionate scraping and search +/// traffic, so they score far above their audience: measured, one carried +/// popularity 313 on **26 votes** and another 242 on **7 votes**, against +/// Bleach's 92 on **2197**. Narrowing the pool to the anime keyword removes the +/// mainstream ballast that keeps them off the front of `/tv/popular`, which is +/// why only the anime rows show it. +/// +/// **Why these five and not `erotic`/`ecchi`.** Those two are *content +/// descriptors* a mainstream show can legitimately carry — Mushoku Tensei +/// (tmdb 94664, 1626 votes) carries both — so excluding them costs real titles. +/// The five here tag adult *productions*, and no false positive was observed. +/// Keyword ids are stable; names resolved via `search/keyword`. +pub(crate) const DISCOVERY_EXCLUDED_KEYWORDS: &str = "198385,155477,325693,445,161919"; + +/// Minimum TMDB vote count for a discovery row entry. +/// +/// The residue filter behind [`DISCOVERY_EXCLUDED_KEYWORDS`]: a handful of +/// erotic-tagged titles carry none of the five excluded keywords, and they are +/// uniformly low-signal (7–37 votes). A vote requires a real logged-in account, +/// which is exactly the signal the traffic-based popularity score lacks, so a +/// floor separates them from genuinely obscure-but-real titles far better than +/// popularity can. +/// +/// 50 is deliberately low. It keeps Ranma½ (169) and Doraemon (237) — it is a +/// noise gate, not a popularity contest. Raising it much further starts costing +/// legitimate niche titles (Akane-banashi 25, Gravitation 17), which is why the +/// keyword exclusion carries the main load and this only sweeps up behind it. +/// +/// Applies to `/discover` rows, which is now every mode except non-anime +/// **trending** — the one signal with no `/discover` equivalent, and therefore +/// the one row that cannot carry this gate. The popular/top-rated modes were +/// moved off `/{kind}/popular` and `/{kind}/top_rated` precisely so they could: +/// three hentai titles were measured at positions 4, 5 and 17 of the +/// *non-anime* `tv/popular` row. +pub(crate) const DISCOVERY_MIN_VOTES: u32 = 50; + /// Default number of cards returned for a free-text query (top-N by TMDB /// popularity). /// @@ -47,6 +111,15 @@ pub(crate) const DEFAULT_CARD_TOP_N: usize = 10; /// never needs a cache wipe — reads slice the cached list to the configured N. pub(crate) const CACHED_PRINCIPAL_DEPTH: usize = 30; +/// Default number of cards a **keyword-less catalog** query returns (a home +/// row: `popular:true contentKind:series fileType:card`). +/// +/// 20 is TMDB's own catalog page size, so the default row costs exactly one +/// TMDB call. Raising it past 20 is legal — [`crate::discovery`] walks further +/// pages — but each page is another call against the shared budget, paid on +/// every warm pass of every row. +pub(crate) const DEFAULT_CARD_DISCOVERY_N: usize = 20; + /// Defensive ceiling on a card's reported season count (guards a corrupt /// `number_of_seasons`). pub(crate) const MAX_CARD_SEASONS: u32 = 50; diff --git a/feeder-plugin/card-feeder/src/discovery.rs b/feeder-plugin/card-feeder/src/discovery.rs new file mode 100644 index 0000000..2fab969 --- /dev/null +++ b/feeder-plugin/card-feeder/src/discovery.rs @@ -0,0 +1,708 @@ +//! Keyword-less catalog discovery: TMDB's popular / trending / top-rated lists +//! → [`Card`]s. +//! +//! The card tier's *browse* surface, as opposed to its search surface. A client +//! with a home page (meta-watch's wall) cannot ask "what should I show?" through +//! a text query — there is no text. This branch answers a query whose intent is +//! carried entirely by structured filters, by calling TMDB's catalog endpoints. +//! +//! ## Query dialect +//! +//! Deliberately identical to the indexer feeder's older `discovery.rs`, which +//! answers the same dialect with metadata-only *seeds*. Same words, so a client +//! switches tiers by swapping `fileType:video` for `fileType:card` and nothing +//! else: +//! +//! - **mode** (one of) — `popular:true` and `top_rated:true` are served by +//! `/discover/{movie,tv}` sorted by popularity / rating (rather than the +//! `/{movie,tv}/popular` list endpoints, which cannot carry the adult gate — +//! see `DISCOVERY_EXCLUDED_KEYWORDS`); `trending:true` → the real +//! `/trending/{movie,tv}/week`, the one mode with no `/discover` equivalent +//! and therefore the one row that stays ungated. +//! - **kind** (required) — `contentKind:movie` → films, +//! `contentKind:series` (or `episode`/`tvshow`/`tv`) → shows. Also what routes +//! the query here at all, alongside `fileType:card`. +//! - **anime** (optional) — `anime:true` narrows `/discover` to TMDB's *anime* +//! keyword, sorted to match the mode. Anime trending has no keyword-filtered +//! endpoint either, so it approximates with popularity — and stays gated. +//! +//! ## One call, complete cards +//! +//! A TMDB catalog list already carries `title`, `overview`, `poster_path` and +//! `year` per entry — everything [`Card::to_record`] needs to emit a renderable +//! card. So a row costs **one** TMDB call, not one plus twenty by-id lookups. +//! +//! The price is a thinner id bag: `tvdbid`/`imdbid` come from `external_ids`, +//! which a list response has no room for, so a discovery card carries only its +//! `tmdbid`. That is the id consumers actually key on (it is what the detail +//! page's phase-2 `tmdbid:` query is built from), and the full bag is resolved +//! anyway the moment someone clicks the card — `card_by_id` is a different code +//! path and still fetches it. Paying twenty extra calls per row up front to +//! pre-fill ids nobody reads until a click would defeat the branch's purpose. +//! +//! ## Why nothing is cached here +//! +//! A catalog list is **mutable by nature** — "popular this week" is the one +//! thing that must not be frozen. The SDK's redb tables are permanent-hit caches +//! sized for immutable data (`external_ids` is explicitly "cached forever"), so +//! persisting a popularity list there would pin the wall to whatever it looked +//! like on first boot. The layer above already de-duplicates the repeat cost: +//! the gateway persists each emitted card to meta-core and its search-coverage +//! gate then skips a repeated `(upstream, query)` for an hour, serving meta-core +//! instead. Every call here is budget-gated, so a burst cannot run away. + +use std::collections::HashMap; +use std::sync::Arc; + +use meta_feeder_sdk::plugin::GatewayQuery; +use tracing::debug; + +use crate::card::{Card, CardSource}; +use crate::consts::{DISCOVERY_EXCLUDED_KEYWORDS, DISCOVERY_MAX_PAGES, DISCOVERY_MIN_VOTES}; +use crate::resolve::{dedup_titles, Resolver}; +use crate::tmdb_client::{TmdbHit, TmdbKind}; + +/// TMDB keyword id for "anime". `/discover` is the only catalog endpoint that +/// takes a keyword filter, which is why `anime:true` switches endpoints rather +/// than adding a parameter. +const TMDB_ANIME_KEYWORD: &str = "210024"; + +/// Results TMDB returns per catalog page — used to translate a desired card +/// count into a page count. +const TMDB_PAGE_SIZE: usize = 20; + +/// The catalog-list flavour a discovery query asks for. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum DiscoveryMode { + Popular, + Trending, + TopRated, +} + +impl DiscoveryMode { + /// The query-filter key that selects this mode. + /// + /// It is also echoed onto every emitted record — not here, but by + /// [`Card::to_record`]'s trailing filter echo, which stamps back every + /// filter the query carried that the card did not already set. That echo is + /// load-bearing: both the gateway dispatcher and meta-search's consumer-side + /// `record_matches` re-apply the query's filters to each record, and a + /// *missing* field fails the match, so an un-echoed `popular` would see + /// every card dropped at one of the two tiers. + fn marker(self) -> &'static str { + match self { + DiscoveryMode::Popular => "popular", + DiscoveryMode::Trending => "trending", + DiscoveryMode::TopRated => "top_rated", + } + } + + /// `sort_by` value for the `/discover` (anime) path. `/discover` has no + /// "trending", so trending approximates to popularity there — the non-anime + /// path still uses the real `/trending` endpoint. + fn discover_sort(self) -> &'static str { + match self { + DiscoveryMode::TopRated => "vote_average.desc", + DiscoveryMode::Popular | DiscoveryMode::Trending => "popularity.desc", + } + } +} + +/// True when this is a keyword-less catalog query this branch should answer. +/// +/// Three conditions, and each excludes a query that belongs to a *different* +/// branch of `handle_query`: no free text (that is the principal search), no +/// `tmdbid:` filter (that is the direct by-id lookup a deep link uses), and a +/// truthy mode marker. Note `contentKind` alone deliberately does **not** select +/// this branch — a text query carries it too. +pub(crate) fn is_discovery_query(query: &GatewayQuery) -> bool { + query.free_text.trim().is_empty() + && !query.filters.contains_key("tmdbid") + && discovery_mode(query).is_some() +} + +/// First truthy mode marker on the query, in precedence order. +fn discovery_mode(query: &GatewayQuery) -> Option { + for mode in [ + DiscoveryMode::Trending, + DiscoveryMode::Popular, + DiscoveryMode::TopRated, + ] { + if filter_is_true(query, mode.marker()) { + return Some(mode); + } + } + None +} + +/// True iff `query.filters[key]` carries a truthy value. +fn filter_is_true(query: &GatewayQuery, key: &str) -> bool { + query + .filters + .get(key) + .is_some_and(|v| v.iter().any(|s| s.eq_ignore_ascii_case("true"))) +} + +/// Map the query's `contentKind` filter to a TMDB media kind. +/// +/// `series` is the canonical spelling for a card (it describes a whole work), +/// but `episode`/`tvshow`/`tv` are accepted so the dialect stays a superset of +/// the indexer feeder's — a client that has not yet switched its row queries +/// still resolves the right endpoint here, even though its records will then be +/// dropped downstream for the mismatch. +fn discovery_kind(query: &GatewayQuery) -> Option { + let values = query.filters.get("contentKind")?; + for v in values { + match v.trim().to_ascii_lowercase().as_str() { + "movie" => return Some(TmdbKind::Movie), + "series" | "episode" | "tvshow" | "tv" => return Some(TmdbKind::Tv), + _ => {} + } + } + None +} + +/// The genre slugs a `genres:` filter requested, folded for comparison. +/// +/// Multi-valued = OR, matching the filter semantics on both re-validating tiers. +fn requested_genres(query: &GatewayQuery) -> Vec { + query + .filters + .get("genres") + .map(|vs| { + vs.iter() + .map(|v| genre_fold(v)) + .filter(|v| !v.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +/// Fold a genre label for comparison: lowercase, alphanumerics only. +/// +/// Must stay identical to `genres_filter_matches`'s fold in the SDK and in +/// meta-search — those two re-apply the filter to every record this emits, so a +/// divergence here means the row's own results get dropped downstream. +fn genre_fold(s: &str) -> String { + s.chars() + .filter(|c| c.is_alphanumeric()) + .flat_map(|c| c.to_lowercase()) + .collect() +} + +/// Resolve the requested genre slugs to TMDB genre ids via the id → name table. +/// Unknown slugs are dropped; if none resolve the caller omits `with_genres` +/// rather than sending an empty filter that TMDB would reject. +fn with_genres_ids(wanted: &[String], names: &HashMap) -> Vec { + let mut ids: Vec = names + .iter() + .filter(|(_, name)| wanted.contains(&genre_fold(name))) + .map(|(id, _)| *id) + .collect(); + ids.sort_unstable(); + ids +} + +/// Build the `path_and_query` for `TmdbClient::discovery_list` from the mode, +/// kind, anime flag, requested genre ids, and 1-based `page`. +fn build_path_and_query( + mode: DiscoveryMode, + kind: TmdbKind, + anime: bool, + genre_ids: &[u32], + page: u32, +) -> String { + let seg = match kind { + TmdbKind::Movie => "movie", + TmdbKind::Tv => "tv", + }; + // Trending is the one mode with no `/discover` equivalent: `/trending` is a + // distinct, TMDB-computed "this week" signal, not a sort order. So it keeps + // the list endpoint — and, being a list endpoint, **cannot carry the adult + // gate below**. The anime variant still routes through `/discover` (there is + // no keyword-filtered trending at all), approximating it with popularity. + // …and a genre-scoped row can't use it either: `/trending` takes no + // `with_genres`, and returning an unfiltered trending page for an "Action" + // row would fill it with whatever is trending. `/discover` sorted by + // popularity is the closest filterable approximation. + if matches!(mode, DiscoveryMode::Trending) && !anime && genre_ids.is_empty() { + return format!("trending/{seg}/week?page={page}"); + } + + // Everything else goes through `/discover`, including the non-anime rows + // that used to use `/{seg}/popular` and `/{seg}/top_rated`. + // + // Those fixed list endpoints accept neither `without_keywords` nor + // `vote_count.gte`, and measurement showed they genuinely need them: three + // hentai titles reached positions 4, 5 and 17 of the *non-anime* `tv/popular` + // row for the same traffic-metric reason described on + // [`DISCOVERY_EXCLUDED_KEYWORDS`]. `/discover` with `sort_by=popularity.desc` + // is the filterable equivalent, so one code path now carries the gate for + // every mode that can hold it. + // + // Top-rated floors higher than the shared gate: sorting by rating puts a + // single 10.0 vote on top, which a 50-vote gate doesn't stop. + let votes = match mode { + DiscoveryMode::TopRated => 200, + _ => DISCOVERY_MIN_VOTES, + }; + let keyword = if anime { + format!("with_keywords={TMDB_ANIME_KEYWORD}&") + } else { + String::new() + }; + // `with_genres` is comma-joined = AND on TMDB's side, pipe-joined = OR. A + // multi-valued filter means OR everywhere else in the DSL, so join with `|` + // to keep the wire semantics and the upstream semantics saying the same + // thing. URL-encoded, since a bare `|` is not valid in a query string. + let genres = if genre_ids.is_empty() { + String::new() + } else { + let joined = genre_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join("%7C"); + format!("with_genres={joined}&") + }; + format!( + "discover/{seg}?{keyword}{genres}sort_by={}&include_adult=false\ + &without_keywords={DISCOVERY_EXCLUDED_KEYWORDS}&vote_count.gte={votes}&page={page}", + mode.discover_sort(), + ) +} + +/// Project one catalog hit into a [`Card`], with no further network calls. +/// +/// `source_id` uses the same `"tv:{id}"` / `"movie:{id}"` shape +/// `Resolver::card_by_id` produces, so a discovery card's `record_id` — and +/// therefore its `0x1007` locator CID — is byte-identical to the one the by-id +/// path derives for the same work. That is what lets a card the wall surfaced +/// and a card the search box surfaced converge on one record instead of two. +fn card_from_hit(hit: &TmdbHit, kind: TmdbKind, genres: &HashMap) -> Card { + let seg = match kind { + TmdbKind::Movie => "movie", + TmdbKind::Tv => "tv", + }; + let mut titles = vec![hit.title.clone()]; + if let Some(o) = &hit.original_title { + titles.push(o.clone()); + } + titles.extend(hit.alt_titles.iter().cloned()); + Card { + source: CardSource::Tmdb, + kind, + source_id: format!("{seg}:{}", hit.tmdbid), + tmdbid: Some(hit.tmdbid), + // Not available on a list response — see the module doc. Resolved on + // the click-time `card_by_id` path instead. + tvdb_id: None, + imdb_id: None, + title: hit.title.clone(), + guard_titles: Arc::new(dedup_titles(titles)), + overview: hit.overview.clone().filter(|s| !s.trim().is_empty()), + poster_path: hit.poster_path.clone().filter(|s| !s.trim().is_empty()), + // A list hit carries bare genre ids; the table names them. + genres: hit.genre_ids.iter().filter_map(|id| genres.get(id).cloned()).collect(), + year: hit.year, + // A list response carries no season structure. `seasonCount` is simply + // omitted from the record (`Card::to_record` skips it at `seasons == 0`). + seasons: 0, + season_summaries: Arc::new(Vec::new()), + } +} + +/// Resolve a keyword-less catalog query to up to `max_results` cards, walking +/// TMDB pages until the cap is reached or a page comes back empty. +/// +/// Best-effort throughout: no mode or no `contentKind` yields `[]`, and a +/// rate-limit or miss breaks the walk and returns what was already gathered. +/// The branch never errors — an empty row is the failure mode, matching +/// `Resolver::cards_for_text`. +pub(crate) async fn cards_for_discovery( + resolver: &Resolver, + query: &GatewayQuery, + max_results: usize, +) -> Vec { + let (Some(mode), Some(kind)) = (discovery_mode(query), discovery_kind(query)) else { + return Vec::new(); + }; + let anime = filter_is_true(query, "anime"); + // Fetched once per process, not per row (see `Resolver::genre_names`). + let gmap = resolver.genre_names(kind).await; + // A `genres:` filter narrows the row upstream via `with_genres`, so the row + // is a full page OF that genre rather than a page filtered down to it. + let wanted = requested_genres(query); + let genre_ids = with_genres_ids(&wanted, &gmap); + if !wanted.is_empty() && genre_ids.is_empty() { + // Asked for a genre this kind doesn't have (e.g. `genres:kids` on the + // movie side, where TMDB files it as "Family"). Returning an unfiltered + // page would silently fill the row with the wrong thing. + debug!( + target: "meta-share::gateway", upstream = "tmdb", + ?wanted, "discovery: no TMDB genre id matched; row left empty" + ); + return Vec::new(); + } + // One page of headroom over the arithmetic minimum: `out` counts *displayable* + // cards, and a catalog page reliably contains a few entries with no overview + // or no poster (newly-added or non-English works), which are declined at + // projection. Without the spare page a 20-card row comes back at 17. + let pages = (max_results.div_ceil(TMDB_PAGE_SIZE).max(1) as u32 + 1).min(DISCOVERY_MAX_PAGES); + let mut out: Vec = Vec::new(); + for page in 1..=pages { + if out.len() >= max_results { + break; + } + let path_and_query = build_path_and_query(mode, kind, anime, &genre_ids, page); + let Some(hits) = resolver.discovery_page(kind, &path_and_query).await else { + break; // budget exhausted, 429, or miss — keep what we have + }; + if hits.is_empty() { + break; // walked past the last catalog page + } + out.extend( + hits.iter() + .map(|h| card_from_hit(h, kind, &gmap)) + .filter(Card::is_displayable), + ); + } + out.truncate(max_results); + debug!( + target: "meta-share::gateway", + upstream = "tmdb", + mode = mode.marker(), + anime, + count = out.len(), + "resolved discovery cards" + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + + fn query(filters: &[(&str, &str)]) -> GatewayQuery { + let mut map: BTreeMap> = BTreeMap::new(); + for (k, v) in filters { + map.entry(k.to_string()).or_default().push(v.to_string()); + } + GatewayQuery { + raw_text: String::new(), + free_text: String::new(), + filters: map, + ranges: Vec::new(), + negations: Vec::new(), + } + } + + #[test] + fn selects_only_keyword_less_mode_queries() { + assert!(is_discovery_query(&query(&[ + ("popular", "true"), + ("contentKind", "series"), + ("fileType", "card"), + ]))); + // A mode marker but with free text → the principal search owns it. + let mut with_text = query(&[("popular", "true"), ("contentKind", "series")]); + with_text.free_text = "frieren".to_string(); + assert!(!is_discovery_query(&with_text)); + // An explicit id → the by-id lookup owns it. + assert!(!is_discovery_query(&query(&[ + ("popular", "true"), + ("tmdbid", "95479"), + ]))); + // contentKind alone must NOT select this branch. + assert!(!is_discovery_query(&query(&[ + ("contentKind", "series"), + ("fileType", "card"), + ]))); + // A falsey marker is not a marker. + assert!(!is_discovery_query(&query(&[("popular", "false")]))); + } + + /// `contentKind:series` is the canonical card spelling; the indexer feeder's + /// older `episode` spelling still resolves the same endpoint. + #[test] + fn content_kind_maps_series_and_the_legacy_spellings() { + assert_eq!( + discovery_kind(&query(&[("contentKind", "series")])), + Some(TmdbKind::Tv) + ); + assert_eq!( + discovery_kind(&query(&[("contentKind", "episode")])), + Some(TmdbKind::Tv) + ); + assert_eq!( + discovery_kind(&query(&[("contentKind", "movie")])), + Some(TmdbKind::Movie) + ); + assert_eq!(discovery_kind(&query(&[("contentKind", "audiobook")])), None); + assert_eq!(discovery_kind(&query(&[])), None); + } + + #[test] + fn builds_the_catalog_endpoints() { + // Popular / top-rated route through /discover (so the gate applies); + // only the sort and the vote floor differ. + let pop = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Movie, false, &[], 1); + assert!(pop.starts_with("discover/movie?"), "{pop}"); + assert!(pop.contains("sort_by=popularity.desc"), "{pop}"); + assert!(pop.contains("page=1"), "{pop}"); + + let paged = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Tv, false, &[], 2); + assert!(paged.starts_with("discover/tv?"), "{paged}"); + assert!(paged.contains("page=2"), "{paged}"); + + let top = build_path_and_query(DiscoveryMode::TopRated, TmdbKind::Movie, false, &[], 1); + assert!(top.contains("sort_by=vote_average.desc"), "{top}"); + assert!(top.contains("vote_count.gte=200"), "{top}"); + + // Trending keeps the real endpoint — see the dedicated test. + assert_eq!( + build_path_and_query(DiscoveryMode::Trending, TmdbKind::Tv, false, &[], 1), + "trending/tv/week?page=1" + ); + } + + #[test] + fn anime_switches_to_the_keyword_filtered_discover_endpoint() { + let p = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Tv, true, &[], 1); + assert!(p.starts_with("discover/tv?"), "{p}"); + assert!(p.contains("with_keywords=210024"), "{p}"); + assert!(p.contains("sort_by=popularity.desc"), "{p}"); + let top = build_path_and_query(DiscoveryMode::TopRated, TmdbKind::Movie, true, &[], 1); + assert!(top.contains("sort_by=vote_average.desc"), "{top}"); + // Sorting by rating puts a 1-vote 10.0 on top, so top-rated floors higher. + assert!(top.contains("vote_count.gte=200"), "{top}"); + } + + /// The gate is not anime-specific. Three hentai titles were measured at + /// positions 4, 5 and 17 of the **non-anime** `tv/popular` row, which is why + /// the plain popular/top-rated modes were moved off the fixed list endpoints + /// (which cannot carry `without_keywords`) and onto `/discover`. + #[test] + fn non_anime_popular_and_top_rated_also_gate() { + for mode in [DiscoveryMode::Popular, DiscoveryMode::TopRated] { + let p = build_path_and_query(mode, TmdbKind::Tv, false, &[], 1); + assert!(p.starts_with("discover/tv?"), "{p}"); + assert!(!p.contains("with_keywords="), "no anime narrowing here: {p}"); + assert!(p.contains("without_keywords="), "{p}"); + assert!(p.contains("vote_count.gte="), "{p}"); + } + } + + /// Trending is the documented exception: `/trending` is a TMDB-computed + /// signal with no `/discover` sort equivalent, so the non-anime trending row + /// keeps the list endpoint and cannot carry the gate. + #[test] + fn non_anime_trending_keeps_the_real_trending_endpoint() { + let p = build_path_and_query(DiscoveryMode::Trending, TmdbKind::Tv, false, &[], 1); + assert_eq!(p, "trending/tv/week?page=1"); + // …but the anime variant has no keyword-filtered trending at all, so it + // still routes through /discover and stays gated. + let a = build_path_and_query(DiscoveryMode::Trending, TmdbKind::Tv, true, &[], 1); + assert!(a.starts_with("discover/tv?"), "{a}"); + assert!(a.contains("without_keywords="), "{a}"); + } + + /// A genre-scoped row narrows **upstream** via `with_genres`, so the row is a + /// full page of that genre rather than a page filtered down to a handful. + #[test] + fn a_genre_row_narrows_upstream() { + let p = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Tv, false, &[10759], 1); + assert!(p.contains("with_genres=10759"), "{p}"); + // Multi-valued = OR, so pipe-joined (comma would mean AND to TMDB). + let multi = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Tv, false, &[16, 35], 1); + assert!(multi.contains("with_genres=16%7C35"), "{multi}"); + // No genre filter ⇒ the parameter is absent, not empty. + let none = build_path_and_query(DiscoveryMode::Popular, TmdbKind::Tv, false, &[], 1); + assert!(!none.contains("with_genres"), "{none}"); + } + + /// A genre row can't use `/trending` (it takes no `with_genres`), so it falls + /// back to the filterable `/discover` rather than returning an unfiltered + /// trending page under a genre heading. + #[test] + fn a_genre_scoped_trending_row_falls_back_to_discover() { + let p = build_path_and_query(DiscoveryMode::Trending, TmdbKind::Tv, false, &[10759], 1); + assert!(p.starts_with("discover/tv?"), "{p}"); + assert!(p.contains("with_genres=10759"), "{p}"); + } + + /// Slugs resolve to TMDB ids through the id → name table, folded on both + /// sides so the DSL never has to spell a space or an ampersand. + #[test] + fn genre_slugs_resolve_to_tmdb_ids() { + let names = HashMap::from([ + (16u32, "Animation".to_string()), + (10759, "Action & Adventure".to_string()), + (10765, "Sci-Fi & Fantasy".to_string()), + ]); + let want = |s: &str| with_genres_ids(&[genre_fold(s)], &names); + assert_eq!(want("action-adventure"), vec![10759]); + assert_eq!(want("Action & Adventure"), vec![10759]); + assert_eq!(want("scififantasy"), vec![10765]); + assert_eq!(want("animation"), vec![16]); + assert!(want("horror").is_empty(), "unknown slug resolves to nothing"); + } + + /// The fold must stay identical to the one both re-validating tiers use, or + /// a row's own results get dropped downstream. + #[test] + fn genre_fold_matches_the_consumer_side_rule() { + assert_eq!(genre_fold("Action & Adventure"), "actionadventure"); + assert_eq!(genre_fold("action-adventure"), "actionadventure"); + assert_eq!(genre_fold("Sci-Fi & Fantasy"), "scififantasy"); + assert_eq!(genre_fold("Science Fiction"), "sciencefiction"); + assert_eq!(genre_fold(" "), ""); + } + + /// The adult-content gate on anime rows, and *why* it is two filters. + /// + /// `include_adult=false` is not enough on its own — measured live, every + /// offending title returns `adult: false` (TMDB reserves that flag for + /// pornography), so the keyword exclusion carries the real load and the vote + /// floor sweeps up the erotic-tagged residue behind it. Dropping either one + /// puts hentai back on the home page. + #[test] + fn anime_rows_gate_adult_content_three_ways() { + for mode in [ + DiscoveryMode::Popular, + DiscoveryMode::Trending, + DiscoveryMode::TopRated, + ] { + for kind in [TmdbKind::Tv, TmdbKind::Movie] { + let p = build_path_and_query(mode, kind, true, &[], 1); + assert!(p.contains("include_adult=false"), "{p}"); + assert!( + p.contains(&format!("without_keywords={DISCOVERY_EXCLUDED_KEYWORDS}")), + "{p}" + ); + assert!(p.contains("vote_count.gte="), "{p}"); + } + } + } + + fn genre_table() -> HashMap { + HashMap::from([(16, "Animation".to_string()), (18, "Drama".to_string())]) + } + + fn hit() -> TmdbHit { + TmdbHit { + tmdbid: 95479, + title: "Frieren: Beyond Journey's End".to_string(), + original_title: Some("葬送のフリーレン".to_string()), + original_language: Some("ja".to_string()), + overview: Some("A mage reflects on her long life.".to_string()), + year: Some(2023), + poster_path: Some("/poster.jpg".to_string()), + genre_ids: vec![16], + alt_titles: Vec::new(), + } + } + + /// A list hit must carry everything a renderable card needs, so the branch + /// costs one TMDB call per page and no by-id follow-ups. + #[test] + fn a_list_hit_becomes_a_complete_card() { + let c = card_from_hit(&hit(), TmdbKind::Tv, &genre_table()); + assert_eq!(c.tmdbid, Some(95479)); + assert!(c.overview.is_some(), "overview is what to_record requires"); + assert!(c.poster_path.is_some(), "poster is what to_record requires"); + assert_eq!(c.year, Some(2023)); + assert_eq!(c.content_kind(), "series"); + // Genres come from the id table — the category axis a card used to lack. + assert_eq!(c.genres, vec!["Animation".to_string()]); + // The id bag beyond tmdbid needs external_ids — deliberately absent. + assert_eq!(c.tvdb_id, None); + assert_eq!(c.imdb_id, None); + } + + /// The locator CID is derived from `record_id`, so a discovery card and a + /// by-id card for the same work must address identically or the two tiers + /// would publish two records for one work. + #[test] + fn record_id_matches_the_by_id_path() { + assert_eq!(card_from_hit(&hit(), TmdbKind::Tv, &genre_table()).record_id(), "tmdb:tv:95479"); + assert_eq!( + card_from_hit(&hit(), TmdbKind::Movie, &genre_table()).record_id(), + "tmdb:movie:95479" + ); + } + + /// The whole point of the branch: `contentKind` on the emitted record is the + /// *card's* kind, never an echo of the query filter. A row asking + /// `contentKind:episode` must still produce `series` here — which is exactly + /// why a client's row query has to ask for `series` (the mismatch is then + /// caught by `record_matches` downstream rather than silently mislabelling). + /// **The invariant this whole branch rests on.** The mode/anime markers are + /// stamped onto the record by `Card::to_record`'s trailing filter echo, not + /// by anything here — and both the gateway dispatcher and meta-search + /// re-apply the query's filters to every record, where a *missing* field + /// fails the match. So if that echo is ever "tidied up", every discovery + /// card is silently dropped in transit and the wall goes blank with no + /// error. This test fails that day. + #[test] + fn a_discovery_card_survives_the_consumer_re_filter() { + let q = query(&[ + ("popular", "true"), + ("contentKind", "series"), + ("fileType", "card"), + ("anime", "true"), + ]); + let card = card_from_hit(&hit(), TmdbKind::Tv, &genre_table()); + let tmdb = crate::tmdb_client::TmdbClient::new("token".to_string()); + let rec = card.to_record(&tmdb, &q.filters).expect("displayable card emits"); + + assert_eq!(rec.fields.get("popular").map(String::as_str), Some("true")); + assert_eq!(rec.fields.get("anime").map(String::as_str), Some("true")); + assert_eq!(rec.fields.get("fileType").map(String::as_str), Some("card")); + assert_eq!( + rec.fields.get("contentKind").map(String::as_str), + Some("series"), + "the card's own kind must win over the query filter" + ); + assert!(rec.fields.contains_key("description/eng")); + assert!(rec.fields.contains_key("poster_url")); + // Genres ride as a key-set, never the legacy comma-joined value + // (METADATA_KEYS §14.12 — new writers must not add csv-set fields). + assert_eq!(rec.fields.get("genres/Animation").map(String::as_str), Some("true")); + assert!(!rec.fields.contains_key("genres"), "must not write the legacy csv-set"); + // The thin id bag: resolved lazily on the click-time by-id path. + assert!(!rec.fields.contains_key("tvdbid")); + assert!(!rec.fields.contains_key("imdbid")); + assert!(!rec.fields.contains_key("seasonCount")); + + assert!( + meta_feeder_sdk::query_eval::record_matches(&rec.fields, &q), + "a discovery card must survive the same filter re-application the \ + dispatcher and meta-search perform" + ); + } + + /// A hit TMDB has not written up yet is declined at projection — which is + /// why the page walk counts displayable cards, not raw hits. + #[test] + fn an_incomplete_hit_is_not_displayable() { + let mut no_overview = hit(); + no_overview.overview = None; + assert!(!card_from_hit(&no_overview, TmdbKind::Tv, &genre_table()).is_displayable()); + + let mut no_poster = hit(); + no_poster.poster_path = None; + assert!(!card_from_hit(&no_poster, TmdbKind::Tv, &genre_table()).is_displayable()); + + assert!(card_from_hit(&hit(), TmdbKind::Tv, &genre_table()).is_displayable()); + } + + #[test] + fn content_kind_comes_from_the_card_not_the_filter() { + let tv = card_from_hit(&hit(), TmdbKind::Tv, &genre_table()); + assert_eq!(tv.content_kind(), "series"); + let movie = card_from_hit(&hit(), TmdbKind::Movie, &genre_table()); + assert_eq!(movie.content_kind(), "movie"); + } +} diff --git a/feeder-plugin/card-feeder/src/lib.rs b/feeder-plugin/card-feeder/src/lib.rs index b281f74..dfd72ec 100644 --- a/feeder-plugin/card-feeder/src/lib.rs +++ b/feeder-plugin/card-feeder/src/lib.rs @@ -12,6 +12,7 @@ pub mod card; pub mod consts; +pub mod discovery; pub mod resolve; pub mod tmdb; pub mod tmdb_budget; diff --git a/feeder-plugin/card-feeder/src/resolve.rs b/feeder-plugin/card-feeder/src/resolve.rs index 16f86e4..feea540 100644 --- a/feeder-plugin/card-feeder/src/resolve.rs +++ b/feeder-plugin/card-feeder/src/resolve.rs @@ -9,6 +9,7 @@ //! The redb table names are unchanged from the indexer feeder's cache, so a //! warm cache carries straight over. +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -16,7 +17,9 @@ use meta_feeder_sdk::cache::MidhashCache; use tracing::debug; use crate::card::{Card, CardSource}; -use crate::consts::{CACHED_PRINCIPAL_DEPTH, TMDB_WAIT_DEADLINE_SECS}; +use crate::consts::{ + CACHED_PRINCIPAL_DEPTH, TMDB_DISCOVERY_WAIT_DEADLINE_SECS, TMDB_WAIT_DEADLINE_SECS, +}; use crate::tmdb_budget::{Lease, TmdbBudget}; use crate::tmdb_client::{ principal_top_n, TmdbCall, TmdbClient, TmdbExternalIds, TmdbHit, TmdbKind, TmdbTvDetails, @@ -28,6 +31,14 @@ pub(crate) struct Resolver { pub(crate) client: Arc, pub(crate) cache: MidhashCache, pub(crate) budget: Arc, + /// `genre id → name` per kind, held for the process lifetime. + /// + /// In memory rather than in the redb cache on purpose: the table is 16 TV / + /// 19 movie entries and effectively static, so a process-local map costs one + /// call per kind per boot and needs no invalidation story — whereas the redb + /// tables are permanent-hit and shared byte-for-byte with the indexer + /// feeder's, which this is not part of. + genre_names: Arc>>>>, } impl Resolver { @@ -36,20 +47,68 @@ impl Resolver { client, cache, budget, + genre_names: Arc::new(std::sync::RwLock::new(HashMap::new())), } } + /// The `genre id → name` table for `kind`, fetched once and then reused. + /// + /// Needed only by the **list** paths (discovery/search), where TMDB gives + /// bare `genre_ids`; a details fetch returns named genre objects directly. + /// Best-effort: a failed or empty fetch yields an empty map (cards simply + /// carry no genres) and is **not** cached, so the next row retries. + pub(crate) async fn genre_names(&self, kind: TmdbKind) -> Arc> { + let key = match kind { + TmdbKind::Tv => "tv", + TmdbKind::Movie => "movie", + }; + if let Some(m) = self.genre_names.read().ok().and_then(|g| g.get(key).cloned()) { + return m; + } + let fetched: HashMap = self + .budgeted_with_deadline( + TMDB_DISCOVERY_WAIT_DEADLINE_SECS, + self.client.genre_list(kind), + |gs| { + gs.into_iter() + .filter(|g| !g.name.trim().is_empty()) + .map(|g| (g.id, g.name)) + .collect() + }, + ) + .await + .unwrap_or_default(); + let arc = Arc::new(fetched); + if !arc.is_empty() { + if let Ok(mut g) = self.genre_names.write() { + g.insert(key, arc.clone()); + } + } + arc + } + /// Acquire a budget token, run `call`, map a hit through `on_hit`. A 429 /// pauses the shared bucket globally rather than being retried here. async fn budgeted( &self, call: impl std::future::Future>, on_hit: impl FnOnce(T) -> R, + ) -> Option { + self.budgeted_with_deadline(TMDB_WAIT_DEADLINE_SECS, call, on_hit) + .await + } + + /// [`Self::budgeted`] with a caller-chosen permit deadline, for callers whose + /// cost of waiting differs from a user-facing lookup's — see + /// [`TMDB_DISCOVERY_WAIT_DEADLINE_SECS`]. + async fn budgeted_with_deadline( + &self, + deadline_secs: u64, + call: impl std::future::Future>, + on_hit: impl FnOnce(T) -> R, ) -> Option { if matches!( - self.budget - .acquire(Duration::from_secs(TMDB_WAIT_DEADLINE_SECS)) - .await, + self.budget.acquire(Duration::from_secs(deadline_secs)).await, Lease::DeadlineExceeded ) { return None; @@ -124,6 +183,29 @@ impl Resolver { .await } + /// One page of a TMDB **catalog list** (popular / trending / top-rated / + /// `discover`) — the keyword-less browse primitive behind + /// [`crate::discovery`]. `None` on a budget timeout, a 429, or a miss, which + /// the caller treats as "stop walking pages and keep what you have". + /// + /// Deliberately **uncached**, unlike every other method here: a catalog list + /// is the one TMDB response that is *supposed* to change under you, and the + /// redb tables in front of the others are permanent-hit caches with no TTL. + /// See the module doc of [`crate::discovery`] for why the repeat cost is + /// already absorbed a layer up. + pub(crate) async fn discovery_page( + &self, + kind: TmdbKind, + path_and_query: &str, + ) -> Option> { + self.budgeted_with_deadline( + TMDB_DISCOVERY_WAIT_DEADLINE_SECS, + self.client.discovery_list(kind, path_and_query), + |hits| hits, + ) + .await + } + /// Cached principal `search/multi`: free text → up to `n` confident /// `(tmdbid, kind)` candidates, most popular first. /// @@ -194,6 +276,7 @@ impl Resolver { guard_titles: Arc::new(dedup_titles(details.guard_titles())), overview: details.overview.clone().filter(|s| !s.is_empty()), poster_path: details.poster_path.clone().filter(|s| !s.is_empty()), + genres: genre_names_of(&details.genres), year, seasons: Card::clamp_seasons(details.number_of_seasons), season_summaries: Arc::new(details.seasons.clone()), @@ -201,6 +284,15 @@ impl Resolver { } TmdbKind::Movie => { let hit = self.movie_hit(tmdbid).await?; + // The movie path round-trips through `TmdbHit` (that is what the + // details cache stores), which keeps genres as bare ids — so + // unlike the TV branch above it needs the id → name table. + let gmap = self.genre_names(TmdbKind::Movie).await; + let genres: Vec = hit + .genre_ids + .iter() + .filter_map(|id| gmap.get(id).cloned()) + .collect(); let mut titles = vec![hit.title.clone()]; if let Some(o) = &hit.original_title { titles.push(o.clone()); @@ -217,6 +309,7 @@ impl Resolver { guard_titles: Arc::new(dedup_titles(titles)), overview: hit.overview.filter(|s| !s.is_empty()), poster_path: hit.poster_path.filter(|s| !s.is_empty()), + genres, year: hit.year, seasons: 0, season_summaries: Arc::new(Vec::new()), @@ -244,8 +337,19 @@ impl Resolver { } } +/// Genre display names off a details payload's `{id, name}` array, trimmed and +/// de-duplicated, order preserved. +pub(crate) fn genre_names_of(genres: &[crate::tmdb_client::TmdbGenre]) -> Vec { + let mut seen = std::collections::HashSet::new(); + genres + .iter() + .map(|g| g.name.trim().to_string()) + .filter(|n| !n.is_empty() && seen.insert(n.to_lowercase())) + .collect() +} + /// Case-insensitive de-dup preserving first-seen order. -fn dedup_titles(titles: Vec) -> Vec { +pub(crate) fn dedup_titles(titles: Vec) -> Vec { let mut seen = std::collections::HashSet::new(); titles .into_iter() diff --git a/feeder-plugin/card-feeder/src/tmdb.rs b/feeder-plugin/card-feeder/src/tmdb.rs index 304aa8d..d726dc6 100644 --- a/feeder-plugin/card-feeder/src/tmdb.rs +++ b/feeder-plugin/card-feeder/src/tmdb.rs @@ -22,7 +22,7 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth}; use tracing::warn; use crate::card::{split_record_id, Card, CardSource}; -use crate::consts::DEFAULT_CARD_TOP_N; +use crate::consts::{DEFAULT_CARD_DISCOVERY_N, DEFAULT_CARD_TOP_N}; use crate::resolve::Resolver; use crate::tmdb_budget::{TmdbBudget, DEFAULT_TMDB_BURST, DEFAULT_TMDB_REFILL_PER_SEC}; use crate::tmdb_client::TmdbClient; @@ -39,6 +39,8 @@ pub struct TmdbConfig { pub tmdb_burst: Option, #[serde(default)] pub card_top_n: Option, + #[serde(default)] + pub card_discovery_n: Option, } impl TmdbConfig { @@ -51,6 +53,9 @@ impl TmdbConfig { .and_then(|v| v.parse().ok()), tmdb_burst: std::env::var("TMDB_BURST").ok().and_then(|v| v.parse().ok()), card_top_n: std::env::var("CARD_TOP_N").ok().and_then(|v| v.parse().ok()), + card_discovery_n: std::env::var("CARD_DISCOVERY_N") + .ok() + .and_then(|v| v.parse().ok()), } } } @@ -62,6 +67,8 @@ pub struct TmdbCardPlugin { /// `depends_on` is satisfied; queries just return nothing. resolver: Option, top_n: usize, + /// Cards per keyword-less catalog row (see [`DEFAULT_CARD_DISCOVERY_N`]). + discovery_n: usize, /// Test hook: `(api_base, image_base)` overriding TMDB's real endpoints, so /// the contract test can point the client at a wiremock server. api_bases: Option<(String, String)>, @@ -79,6 +86,7 @@ impl TmdbCardPlugin { config: TmdbConfig::from_env(), resolver: None, top_n: DEFAULT_CARD_TOP_N, + discovery_n: DEFAULT_CARD_DISCOVERY_N, api_bases: None, } } @@ -111,6 +119,9 @@ impl TmdbCardPlugin { if file_cfg.card_top_n.is_some() { self.config.card_top_n = file_cfg.card_top_n; } + if file_cfg.card_discovery_n.is_some() { + self.config.card_discovery_n = file_cfg.card_discovery_n; + } } } } @@ -142,6 +153,11 @@ impl FeederPlugin for TmdbCardPlugin { self.load_config(cache_dir); let cache: MidhashCache = meta_feeder_sdk::common::open_midhash_cache(cache_dir, "tmdb")?; self.top_n = self.config.card_top_n.filter(|n| *n > 0).unwrap_or(DEFAULT_CARD_TOP_N); + self.discovery_n = self + .config + .card_discovery_n + .filter(|n| *n > 0) + .unwrap_or(DEFAULT_CARD_DISCOVERY_N); let token = self.config.tmdb_token.trim().to_string(); if token.is_empty() { @@ -189,15 +205,33 @@ impl FeederPlugin for TmdbCardPlugin { self.served_file_types(), self.served_content_kinds(), ) { + // One misconfiguration lands here often enough to deserve a name. + // A browse row written against the *indexer* feeder's dialect says + // `contentKind:episode`, which is not in `served_content_kinds` — so + // the row is rejected here, before TMDB is touched, and the operator + // sees an empty wall with no error anywhere. A card describes a WORK, + // and a work is a `series`, never an `episode`. + if crate::discovery::is_discovery_query(query) { + warn!( + target: "meta-share::gateway", + upstream = "tmdb", + content_kind = ?query.filters.get("contentKind"), + "discovery row rejected by the served-kinds gate: a card is a \ + WORK — use contentKind:series for TV rows (not episode/tv/tvshow)" + ); + } return Ok(Vec::new()); } let Some(resolver) = self.resolver() else { return Ok(Vec::new()); // soft-skipped (no token) }; - // An explicit `tmdbid:` filter resolves exactly one card — the direct - // "give me this work" lookup a deep link uses. Otherwise the free text - // goes through the principal search. + // Three shapes, in precedence order: + // 1. an explicit `tmdbid:` filter — the direct "give me this work" + // lookup a deep link (or the detail page) uses; + // 2. a keyword-less catalog query (`popular:true contentKind:series`) + // — a client's home row, answered from TMDB's catalog endpoints; + // 3. free text — the principal search. let cards: Vec = if let Some(id) = query .filters .get("tmdbid") @@ -206,6 +240,9 @@ impl FeederPlugin for TmdbCardPlugin { { let kind = kind_hint(query, resolver, id).await; resolver.card_by_id(id, kind).await.into_iter().collect() + } else if crate::discovery::is_discovery_query(query) { + let n = self.discovery_n.min(max_results.max(1)); + crate::discovery::cards_for_discovery(resolver, query, n).await } else { let free_text = query.free_text.trim(); if free_text.is_empty() { @@ -284,6 +321,13 @@ impl FeederPlugin for TmdbCardPlugin { Blank keeps the built-in default (10). Unlike the old anchor \ top-N this costs no indexer requests — only TMDB.", ), + F::number("card_discovery_n", "Cards per discovery row").with_help( + "How many cards a keyword-less browse row (popular / trending / \ + top-rated) returns. Blank keeps the built-in default (20 — \ + exactly one TMDB catalog page, so a row costs ONE request). \ + Separate from 'Cards per search': a row is a browse surface, \ + not an answer to a question.", + ), ], } } @@ -294,6 +338,7 @@ impl FeederPlugin for TmdbCardPlugin { "tmdb_rate_per_sec": self.config.tmdb_rate_per_sec, "tmdb_burst": self.config.tmdb_burst, "card_top_n": self.config.card_top_n, + "card_discovery_n": self.config.card_discovery_n, }) } } diff --git a/feeder-plugin/card-feeder/src/tmdb_client.rs b/feeder-plugin/card-feeder/src/tmdb_client.rs index b2ca587..f0b67ba 100644 --- a/feeder-plugin/card-feeder/src/tmdb_client.rs +++ b/feeder-plugin/card-feeder/src/tmdb_client.rs @@ -309,6 +309,27 @@ impl TmdbClient { } } + /// `GET /3/genre/{tv,movie}/list` — the id → name table for the `genre_ids` + /// a discovery/search list hit carries. + /// + /// TMDB returns genres as bare ids on list responses and as `{id, name}` + /// objects only on a details fetch, so the *browse* path has no way to name + /// a genre without this table. It is tiny (16 TV / 19 movie entries) and + /// effectively static, which is why the caller holds it for the process + /// lifetime instead of paying it per row. + pub(crate) async fn genre_list(&self, kind: TmdbKind) -> TmdbCall> { + let seg = match kind { + TmdbKind::Movie => "movie", + TmdbKind::Tv => "tv", + }; + let url = format!("{}/genre/{seg}/list", self.api_base.trim_end_matches('/')); + match self.get_json::(&url, "genre_list").await { + TmdbCall::Hit(r) => TmdbCall::Hit(r.genres), + TmdbCall::Miss => TmdbCall::Miss, + TmdbCall::RateLimited(d) => TmdbCall::RateLimited(d), + } + } + /// Shared `GET ` → JSON helper with the standard bearer auth, search /// timeout, 429→`RateLimited`, non-2xx/transient→`Miss` contract. Factored /// out of [`tv_details`]/[`external_ids`]/[`movie_details`] (identical @@ -393,6 +414,21 @@ pub(crate) enum TmdbKind { Tv, } +/// One `{id, name}` genre entry, as returned by `genre/{kind}/list` and by the +/// details endpoints' `genres` array. +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct TmdbGenre { + pub(crate) id: u32, + #[serde(default)] + pub(crate) name: String, +} + +#[derive(serde::Deserialize)] +struct TmdbGenreList { + #[serde(default)] + genres: Vec, +} + #[derive(serde::Deserialize)] pub(crate) struct TmdbSearchResponse { #[serde(default)] @@ -550,6 +586,10 @@ impl TmdbHit { pub(crate) struct TmdbTvDetails { #[serde(default)] pub(crate) number_of_seasons: u32, + /// Genre objects (`{id, name}`) — present on a details fetch, unlike the + /// bare `genre_ids` a list response carries. Free here: same payload. + #[serde(default)] + pub(crate) genres: Vec, /// One entry per season TMDB knows about, including season 0 /// ("Specials"). `episode_count` is the released-episode total. #[serde(default)] @@ -691,6 +731,9 @@ pub(crate) struct TmdbExternalIds { #[derive(Debug, Clone, serde::Deserialize)] pub(crate) struct TmdbMovieDetails { pub(crate) id: u64, + /// See [`TmdbTvDetails::genres`]. + #[serde(default)] + pub(crate) genres: Vec, #[serde(default)] pub(crate) title: Option, #[serde(default)]