Worph
@@ -0,0 +1,42 @@
|
||||
[package]
|
||||
name = "card-feeder"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
|
||||
# Card feeder sidecar: the DISCOVERY half of gateway search. Bridges metadata
|
||||
# sources (TMDB today; MyAnimeList later, as a second FeederPlugin in this same
|
||||
# binary) into `fileType=card` records — poster + description + an id bag
|
||||
# describing a WORK, never a file. It serves no bytes and never touches an
|
||||
# indexer: retrieval is a separate, second query answered by the content
|
||||
# feeders. See ../../../meta-gateway/docs/others/card-tier-search.md.
|
||||
|
||||
[lib]
|
||||
name = "card_feeder"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "card-feeder"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" }
|
||||
|
||||
tokio = { version = "1.40", features = ["full"] }
|
||||
axum = { version = "0.7", features = ["json"] }
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
|
||||
bytes = "1"
|
||||
futures = "0.3"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
|
||||
[dev-dependencies]
|
||||
wiremock = "0.6"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.40", features = ["full", "test-util"] }
|
||||
@@ -0,0 +1,56 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# card-feeder — metadata-bridge feeder sidecar (the card tier). Multi-stage.
|
||||
#
|
||||
# The build context must be the REPO ROOT (the workspace) so the
|
||||
# `meta-feeder-sdk` path dependency (../../crates/meta-feeder-sdk) resolves.
|
||||
# Same BuildKit cache-mount strategy as the other feeders: the cargo
|
||||
# registry/git and `target/` persist across builds; the release binary is
|
||||
# copied OUT of the target cache mount within the same RUN.
|
||||
# =============================================================================
|
||||
|
||||
FROM rust:1.89-slim-bookworm AS builder
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ca-certificates \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY Cargo.toml rust-toolchain.toml ./
|
||||
COPY Cargo.lock* ./
|
||||
COPY crates/ crates/
|
||||
COPY feeder-plugin/ feeder-plugin/
|
||||
|
||||
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=meta-feeder-card-target,target=/build/target \
|
||||
cargo build --release --bin card-feeder \
|
||||
&& cp target/release/card-feeder /card-feeder
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /card-feeder /usr/local/bin/card-feeder
|
||||
|
||||
RUN useradd --system --uid 10001 --no-create-home metafeeder \
|
||||
&& mkdir -p /data/meta-feeder \
|
||||
&& chown metafeeder:metafeeder /data/meta-feeder
|
||||
|
||||
USER metafeeder
|
||||
WORKDIR /data/meta-feeder
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
|
||||
META_FEEDER_STATE_DIR=/data/meta-feeder \
|
||||
RUST_LOG=info
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/card-feeder"]
|
||||
@@ -0,0 +1,342 @@
|
||||
//! The card model and its projection to a `DiscoveryRecord`.
|
||||
//!
|
||||
//! A **card** identifies a *work* — a series, a film — not a file. It carries
|
||||
//! the display trio meta-watch's quality gate needs (title, poster,
|
||||
//! description) plus an **id bag**: every cross-source identifier the metadata
|
||||
//! bridge could resolve. Nothing here is a byte locator; the card's own CID is
|
||||
//! a `0x1007` card-locator derived from `(source, id)` and nothing is ever
|
||||
//! fetchable by it.
|
||||
//!
|
||||
//! This is the lifted-and-renamed `ResolvedAnchor` from the indexer feeder's
|
||||
//! `torznab/mod.rs`, promoted from a private query-time struct to the feeder's
|
||||
//! public output. The projection below is its `build_anchor_record`, minus the
|
||||
//! two fields that only made sense while a card was pretending to be a video
|
||||
//! (`anchorPlaceholder`, `videoType`) — see
|
||||
//! `meta-gateway/docs/others/card-tier-search.md` §2.1.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use meta_feeder_sdk::hash::compute_card_cid;
|
||||
use meta_feeder_sdk::types::DiscoveryRecord;
|
||||
|
||||
use crate::consts::MAX_CARD_SEASONS;
|
||||
use crate::tmdb_client::{TmdbClient, TmdbKind, TmdbSeasonSummary};
|
||||
|
||||
/// The metadata source that published a card. One variant per bridge; the
|
||||
/// string form is the CID's source namespace, so **changing it re-mints every
|
||||
/// card CID from that source** — treat these as wire constants.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum CardSource {
|
||||
Tmdb,
|
||||
}
|
||||
|
||||
impl CardSource {
|
||||
pub(crate) fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
CardSource::Tmdb => "tmdb",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A resolved work. Immutable by contract: cards carry no rating, popularity,
|
||||
/// or any other value that drifts, precisely so the identity and the content
|
||||
/// stay in agreement forever (design doc §5). A newly aired season is a *new*
|
||||
/// record, never an edit to this one.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct Card {
|
||||
pub(crate) source: CardSource,
|
||||
pub(crate) kind: TmdbKind,
|
||||
/// Source-local id for the work — `"tv:95479"` / `"movie:27205"` for TMDB.
|
||||
/// Half of the card CID's preimage; see [`Card::record_id`].
|
||||
pub(crate) source_id: String,
|
||||
|
||||
// -- the id bag ---------------------------------------------------------
|
||||
// Every cross-source identifier resolved for this work. The phase-2 query
|
||||
// selects the best one for a given indexer's capabilities rather than
|
||||
// assuming `tmdbid`, so a future MyAnimeList-only card needs no new filter
|
||||
// name on the wire (design doc §10b).
|
||||
pub(crate) tmdbid: Option<u64>,
|
||||
/// `i64` to match TMDB's `external_ids` payload verbatim — no lossy cast.
|
||||
pub(crate) tvdb_id: Option<i64>,
|
||||
pub(crate) imdb_id: Option<String>,
|
||||
|
||||
// -- display ------------------------------------------------------------
|
||||
pub(crate) title: String,
|
||||
/// Canonical + original + AKA titles. Rides along so the *content* feeders
|
||||
/// can run their resemblance guard against the card instead of re-querying
|
||||
/// TMDB for it.
|
||||
///
|
||||
/// Not read here yet — it is consumed by phase 2, which still resolves its
|
||||
/// own anchor until the indexer feeder's `build_anchor_record` path is
|
||||
/// retired. Resolving it now means the card is already complete when that
|
||||
/// switch happens.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) guard_titles: Arc<Vec<String>>,
|
||||
pub(crate) overview: Option<String>,
|
||||
pub(crate) poster_path: Option<String>,
|
||||
pub(crate) year: Option<u16>,
|
||||
|
||||
// -- structure (TV only) ------------------------------------------------
|
||||
pub(crate) seasons: u32,
|
||||
/// Per-season episode counts. Surfaced on the record only as `seasonCount`
|
||||
/// today; the full summaries are what a *season-scoped* phase 2 would fan
|
||||
/// out over. Deliberately descoped for v1 (design doc §12.1) — note that a
|
||||
/// season list is the one thing an immutable card cannot carry forever, so
|
||||
/// when that lands it comes either from season cards or a live lookup, not
|
||||
/// from a mutated series card.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) season_summaries: Arc<Vec<TmdbSeasonSummary>>,
|
||||
}
|
||||
|
||||
impl Card {
|
||||
/// `contentKind` for this card: a work-level facet, orthogonal to
|
||||
/// `fileType=card`. A whole show is `series` (an individual instalment
|
||||
/// would be `episode`, which a card never is).
|
||||
pub(crate) fn content_kind(&self) -> &'static str {
|
||||
match self.kind {
|
||||
TmdbKind::Tv => "series",
|
||||
TmdbKind::Movie => "movie",
|
||||
}
|
||||
}
|
||||
|
||||
/// The feeder-scoped record id, `"<source>:<source_id>"` — also the exact
|
||||
/// preimage half of the card CID, so `compute_outcomes` can round-trip a
|
||||
/// record id straight back to a CID with no lookup.
|
||||
pub(crate) fn record_id(&self) -> String {
|
||||
format!("{}:{}", self.source.as_str(), self.source_id)
|
||||
}
|
||||
|
||||
/// This card's `0x1007` locator CID. Deterministic in `(source, id)`, so
|
||||
/// any peer holding a tmdb id derives the same address offline.
|
||||
///
|
||||
/// `compute_outcomes` doesn't use this — it derives the CID straight from
|
||||
/// the record id string (`split_record_id`), deliberately, so deriving an
|
||||
/// address never needs a resolved `Card`. [`Card::to_record`] does use it,
|
||||
/// to stamp the `cids/` key-set member (see there for why that matters).
|
||||
pub(crate) fn cid(&self) -> Option<String> {
|
||||
compute_card_cid(self.source.as_str(), &self.source_id)
|
||||
}
|
||||
|
||||
/// Clamp a TMDB season count into the defensive ceiling.
|
||||
pub(crate) fn clamp_seasons(n: u32) -> u32 {
|
||||
n.clamp(1, MAX_CARD_SEASONS)
|
||||
}
|
||||
|
||||
/// Project to the wire record.
|
||||
///
|
||||
/// Returns `None` when the card lacks a poster or a description — it would
|
||||
/// fail meta-watch's quality gate and render as a hole in the grid, so the
|
||||
/// feeder declines to emit it at all rather than shipping an unrenderable
|
||||
/// card.
|
||||
///
|
||||
/// `query_filters` echoes back every structured filter the query carried
|
||||
/// (except `languages`, which fails open on a record that has none) so the
|
||||
/// record survives the gateway's and meta-search's `record_matches`
|
||||
/// re-filter on the way out.
|
||||
pub(crate) fn to_record(
|
||||
&self,
|
||||
tmdb: &TmdbClient,
|
||||
query_filters: &BTreeMap<String, Vec<String>>,
|
||||
) -> Option<DiscoveryRecord> {
|
||||
let overview = self.overview.as_deref().map(str::trim).filter(|s| !s.is_empty())?;
|
||||
let poster_path = self.poster_path.as_deref().filter(|s| !s.is_empty())?;
|
||||
|
||||
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
||||
// Type axes. `card` on the fileType axis and the work kind on the
|
||||
// contentKind axis stay orthogonal, so `fileType:card contentKind:series`
|
||||
// is a well-formed query and routing needs no special case.
|
||||
fields.insert("fileType".to_string(), "card".to_string());
|
||||
fields.insert("contentKind".to_string(), self.content_kind().to_string());
|
||||
fields.insert("title".to_string(), self.title.clone());
|
||||
|
||||
// The id bag.
|
||||
if let Some(id) = self.tmdbid {
|
||||
fields.insert("tmdbid".to_string(), id.to_string());
|
||||
}
|
||||
if let Some(id) = self.tvdb_id {
|
||||
fields.insert("tvdbid".to_string(), id.to_string());
|
||||
}
|
||||
if let Some(id) = self.imdb_id.as_deref().filter(|s| !s.is_empty()) {
|
||||
fields.insert("imdbid".to_string(), id.to_string());
|
||||
}
|
||||
if let Some(y) = self.year {
|
||||
fields.insert("movieYear".to_string(), y.to_string());
|
||||
}
|
||||
if self.kind == TmdbKind::Tv && self.seasons > 0 {
|
||||
fields.insert("seasonCount".to_string(), self.seasons.to_string());
|
||||
}
|
||||
|
||||
// Synopsis under the namespaced key meta-search indexes (not a flat
|
||||
// `overview`).
|
||||
fields.insert("description/eng".to_string(), overview.to_string());
|
||||
|
||||
// The bare-CID key-set member, exactly as every other feeder stamps on
|
||||
// its search records (`indexer-feeder`'s `torznab/xml.rs` does the same
|
||||
// with the release's infohash).
|
||||
//
|
||||
// **Load-bearing, and not merely cosmetic.** The gateway's search path
|
||||
// persists a hit to meta-core only if `url_key_for` can find a CID on
|
||||
// it (`dispatch.rs` — no key ⇒ `continue`, silently unpersisted). A card
|
||||
// is the one record type whose CID isn't discovered from an upstream but
|
||||
// *derived*, and `compute_outcomes` (the other place it's derived) is not
|
||||
// on the search path — so without this line a card is never persisted at
|
||||
// all. That interacts badly with the search-coverage gate, which stamps
|
||||
// `(upstream, query)` as covered whenever the feeder produced records and
|
||||
// thereafter skips the feeder for an hour, serving meta-core instead:
|
||||
// coverage marked + nothing persisted = **an identical card search
|
||||
// returns empty for the rest of the window**. Observed live before this
|
||||
// was added.
|
||||
//
|
||||
// Stamping it also delivers §4's addressability and §9.4's free
|
||||
// cross-peer convergence: two peers that resolve the same work publish
|
||||
// the same locator, so meta-core's reverse index folds them into one
|
||||
// record with no merge logic.
|
||||
if let Some(cid) = self.cid() {
|
||||
fields.insert(format!("cids/{cid}"), "true".to_string());
|
||||
}
|
||||
|
||||
// Poster: the transient `posterPath` is rewritten to `poster_url` below,
|
||||
// and the GATEWAY CORE seeds that URL into a content-addressed `poster`
|
||||
// cid (`blockstore::seed_preview`). The feeder never fetches poster
|
||||
// bytes — invariant 10.
|
||||
fields.insert(
|
||||
"poster_url".to_string(),
|
||||
tmdb.poster_cdn_url(poster_path),
|
||||
);
|
||||
|
||||
for (key, allowed) in query_filters {
|
||||
if key == "languages" || fields.contains_key(key) || allowed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
fields.insert(key.clone(), allowed.join(","));
|
||||
}
|
||||
|
||||
Some(DiscoveryRecord {
|
||||
upstream_id: self.source.as_str().to_string(),
|
||||
record_id: self.record_id(),
|
||||
fields,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a card `record_id` (`"tmdb:tv:95479"`) back into `(source, source_id)`
|
||||
/// — the two halves of the CID preimage. `None` when the id carries no source
|
||||
/// prefix.
|
||||
pub(crate) fn split_record_id(record_id: &str) -> Option<(&str, &str)> {
|
||||
record_id.split_once(':').filter(|(s, id)| !s.is_empty() && !id.is_empty())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tv_card() -> Card {
|
||||
Card {
|
||||
source: CardSource::Tmdb,
|
||||
kind: TmdbKind::Tv,
|
||||
source_id: "tv:95479".to_string(),
|
||||
tmdbid: Some(95479),
|
||||
tvdb_id: Some(367189),
|
||||
imdb_id: Some("tt22248376".to_string()),
|
||||
title: "Frieren: Beyond Journey's End".to_string(),
|
||||
guard_titles: Arc::new(vec!["Sousou no Frieren".to_string()]),
|
||||
overview: Some("The story follows the elf mage Frieren.".to_string()),
|
||||
poster_path: Some("/dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg".to_string()),
|
||||
year: Some(2023),
|
||||
seasons: 1,
|
||||
season_summaries: Arc::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_id_round_trips_to_the_cid_preimage() {
|
||||
let card = tv_card();
|
||||
let record_id = card.record_id();
|
||||
assert_eq!(record_id, "tmdb:tv:95479");
|
||||
let (source, id) = split_record_id(&record_id).unwrap();
|
||||
assert_eq!((source, id), ("tmdb", "tv:95479"));
|
||||
// The id a peer would derive offline must equal the one we publish.
|
||||
assert_eq!(compute_card_cid(source, id), card.cid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_carries_both_type_axes_and_the_whole_id_bag() {
|
||||
let tmdb = TmdbClient::new("token".to_string());
|
||||
let rec = tv_card().to_record(&tmdb, &BTreeMap::new()).expect("record");
|
||||
assert_eq!(rec.fields["fileType"], "card");
|
||||
assert_eq!(rec.fields["contentKind"], "series");
|
||||
assert_eq!(rec.fields["tmdbid"], "95479");
|
||||
assert_eq!(rec.fields["tvdbid"], "367189");
|
||||
assert_eq!(rec.fields["imdbid"], "tt22248376");
|
||||
assert!(rec.fields["poster_url"].contains("dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg"));
|
||||
assert!(rec.fields.contains_key("description/eng"));
|
||||
// A card is not a video pretending to have no stream — it is its own
|
||||
// type. The two markers the placeholder needed are gone.
|
||||
assert!(!rec.fields.contains_key("videoType"));
|
||||
assert!(!rec.fields.contains_key("anchorPlaceholder"));
|
||||
// And it must never carry a poster *path* (the transient field the
|
||||
// gateway core would not know how to seed).
|
||||
assert!(!rec.fields.contains_key("posterPath"));
|
||||
}
|
||||
|
||||
/// Regression: the record MUST carry a `cids/` key-set member, because the
|
||||
/// gateway's search-persist path silently drops any record `url_key_for`
|
||||
/// finds no CID on. Without it a card is never persisted, and the
|
||||
/// search-coverage gate then blanks every repeat search for an hour (see
|
||||
/// the comment in `to_record`). This failed live before the fix.
|
||||
#[test]
|
||||
fn record_carries_its_locator_as_a_cids_keyset_member() {
|
||||
let tmdb = TmdbClient::new("token".to_string());
|
||||
let card = tv_card();
|
||||
let rec = card.to_record(&tmdb, &BTreeMap::new()).expect("record");
|
||||
let cid = card.cid().expect("locator");
|
||||
assert_eq!(
|
||||
rec.fields.get(&format!("cids/{cid}")).map(String::as_str),
|
||||
Some("true"),
|
||||
"a card must be persistable; see to_record's cids/ comment"
|
||||
);
|
||||
// And it must be the derived locator, not some other cid shape.
|
||||
assert!(cid.starts_with('b'));
|
||||
}
|
||||
|
||||
/// The quality gate lives here, not in the consumer: a card with no poster
|
||||
/// or no synopsis would render as a hole in meta-watch's grid.
|
||||
#[test]
|
||||
fn card_without_poster_or_overview_is_not_emitted() {
|
||||
let tmdb = TmdbClient::new("token".to_string());
|
||||
let mut no_poster = tv_card();
|
||||
no_poster.poster_path = None;
|
||||
assert!(no_poster.to_record(&tmdb, &BTreeMap::new()).is_none());
|
||||
|
||||
let mut no_overview = tv_card();
|
||||
no_overview.overview = Some(" ".to_string());
|
||||
assert!(no_overview.to_record(&tmdb, &BTreeMap::new()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn movie_card_uses_the_movie_content_kind() {
|
||||
let mut card = tv_card();
|
||||
card.kind = TmdbKind::Movie;
|
||||
card.source_id = "movie:27205".to_string();
|
||||
assert_eq!(card.content_kind(), "movie");
|
||||
assert_eq!(card.record_id(), "tmdb:movie:27205");
|
||||
// Distinct id namespace → distinct CID from the tv card.
|
||||
assert_ne!(card.cid(), tv_card().cid());
|
||||
}
|
||||
|
||||
/// Query filters are echoed so the record survives `record_matches` on the
|
||||
/// way out, but `languages` is deliberately skipped (it fails open).
|
||||
#[test]
|
||||
fn query_filters_are_echoed_except_languages() {
|
||||
let tmdb = TmdbClient::new("token".to_string());
|
||||
let mut filters = BTreeMap::new();
|
||||
filters.insert("fileType".to_string(), vec!["card".to_string()]);
|
||||
filters.insert("genres".to_string(), vec!["fantasy".to_string()]);
|
||||
filters.insert("languages".to_string(), vec!["jpn".to_string()]);
|
||||
let rec = tv_card().to_record(&tmdb, &filters).expect("record");
|
||||
assert_eq!(rec.fields["genres"], "fantasy");
|
||||
assert_eq!(rec.fields["fileType"], "card", "own value wins the echo");
|
||||
assert!(!rec.fields.contains_key("languages"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//! Constants for the card feeder. The TMDB subset of the indexer feeder's
|
||||
//! `consts.rs` — lifted verbatim so a warm redb cache and the observed live
|
||||
//! behaviour carry over unchanged — plus the card-tier's own anchor knobs.
|
||||
|
||||
/// User-Agent for outbound HTTP from the card feeder.
|
||||
pub(crate) const USER_AGENT: &str = concat!(
|
||||
"meta-card/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (gateway:card)"
|
||||
);
|
||||
|
||||
/// TMDB REST API base (v3).
|
||||
pub(crate) const TMDB_API_BASE: &str = "https://api.themoviedb.org/3";
|
||||
/// TMDB image CDN base; callers append a size segment (e.g. `/w500/<path>`).
|
||||
pub(crate) const TMDB_IMAGE_BASE: &str = "https://image.tmdb.org/t/p";
|
||||
/// TMDB poster size segment inserted between [`TMDB_IMAGE_BASE`] and the
|
||||
/// poster path. Mandatory — TMDB 404s a path with no size. `w500` is the
|
||||
/// poster-grid sweet spot (≈122 KB vs `original`'s multi-MB).
|
||||
pub(crate) const TMDB_POSTER_SIZE: &str = "w500";
|
||||
|
||||
/// TMDB search request timeout (seconds).
|
||||
pub(crate) const TMDB_SEARCH_TIMEOUT_SECS: u64 = 1000;
|
||||
/// TMDB poster download timeout (seconds). The card feeder never downloads
|
||||
/// poster bytes (the gateway core seeds them), but the client is shared with
|
||||
/// the indexer feeder's lift, so the knob stays.
|
||||
pub(crate) const TMDB_POSTER_TIMEOUT_SECS: u64 = 3000;
|
||||
/// Default Retry-After (seconds) applied when TMDB returns HTTP 429 with no
|
||||
/// (or an unparseable) `Retry-After` header.
|
||||
pub(crate) const TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS: u64 = 5;
|
||||
/// Maximum time to wait for a TMDB budget permit before giving up (seconds).
|
||||
pub(crate) const TMDB_WAIT_DEADLINE_SECS: u64 = 3000;
|
||||
|
||||
/// Default number of cards returned for a free-text query (top-N by TMDB
|
||||
/// popularity).
|
||||
///
|
||||
/// **This is no longer a fan-out multiplier.** In the pre-card design the same
|
||||
/// knob (`DEFAULT_ANCHOR_TOP_N`) multiplied the indexer request count — every
|
||||
/// anchor became its own paged search per indexer against a shared 1 req/s
|
||||
/// bucket, so ten anchors meant the show the user actually wanted got a tenth
|
||||
/// of the budget. A card query touches TMDB only, so N here costs at most one
|
||||
/// cached `search/multi`; the indexer budget is spent later, entirely on the
|
||||
/// single card the user clicks.
|
||||
pub(crate) const DEFAULT_CARD_TOP_N: usize = 10;
|
||||
|
||||
/// How many ranked principal-search anchors to persist per query in the redb
|
||||
/// cache. Sized comfortably above any reasonable top-N so flipping the knob
|
||||
/// never needs a cache wipe — reads slice the cached list to the configured N.
|
||||
pub(crate) const CACHED_PRINCIPAL_DEPTH: usize = 30;
|
||||
|
||||
/// Defensive ceiling on a card's reported season count (guards a corrupt
|
||||
/// `number_of_seasons`).
|
||||
pub(crate) const MAX_CARD_SEASONS: u32 = 50;
|
||||
@@ -0,0 +1,18 @@
|
||||
//! `card-feeder` library surface.
|
||||
//!
|
||||
//! The **discovery** half of gateway search. Bridges metadata sources into
|
||||
//! `fileType=card` records — poster + description + an id bag describing a
|
||||
//! *work*, never a file — so the question "which work is this?" is answered
|
||||
//! cheaply and separately from "which releases exist for it?".
|
||||
//!
|
||||
//! One upstream today (`tmdb`). MyAnimeList will be a sibling `FeederPlugin`
|
||||
//! in this same binary, declaring the same `served_file_types = ["card"]`.
|
||||
//!
|
||||
//! Design: `meta-gateway/docs/others/card-tier-search.md`.
|
||||
|
||||
pub mod card;
|
||||
pub mod consts;
|
||||
pub mod resolve;
|
||||
pub mod tmdb;
|
||||
pub mod tmdb_budget;
|
||||
pub mod tmdb_client;
|
||||
@@ -0,0 +1,33 @@
|
||||
//! `card-feeder` — metadata-bridge feeder sidecar (the card tier).
|
||||
//!
|
||||
//! Hosts [`tmdb::TmdbCardPlugin`] over the feeder HTTP contract via
|
||||
//! [`meta_feeder_sdk::serve_feeders`]. The gateway core reaches it
|
||||
//! server-to-server on the internal docker network.
|
||||
//!
|
||||
//! Env:
|
||||
//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
|
||||
//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`)
|
||||
//! - `TMDB_TOKEN` — TMDB v4 read token (seed; the config page wins)
|
||||
//! - `TMDB_RATE_PER_SEC`, `TMDB_BURST`, `CARD_TOP_N` — optional seeds
|
||||
//! - `RUST_LOG` — tracing filter (default `info`)
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use card_feeder::tmdb::TmdbCardPlugin;
|
||||
use meta_feeder_sdk::serve_feeders;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
|
||||
.init();
|
||||
|
||||
let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string())
|
||||
.parse()?;
|
||||
let state_dir =
|
||||
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
||||
|
||||
serve_feeders(vec![Box::new(TmdbCardPlugin::new())], state_dir, listen).await
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
//! Cache- and budget-gated TMDB resolution: free text → [`Card`]s.
|
||||
//!
|
||||
//! A slimmed lift of the indexer feeder's `enrich.rs` `TmdbEnricher`. Only the
|
||||
//! **anchor-resolution** half comes across — `principal_top_n`, `tv_details`,
|
||||
//! `movie_details`, `external_ids`. The per-release fuzzy title search and its
|
||||
//! single-flight map stay in the indexer feeder, which still owns release
|
||||
//! enrichment (design doc §8).
|
||||
//!
|
||||
//! The redb table names are unchanged from the indexer feeder's cache, so a
|
||||
//! warm cache carries straight over.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use meta_feeder_sdk::cache::MidhashCache;
|
||||
use tracing::debug;
|
||||
|
||||
use crate::card::{Card, CardSource};
|
||||
use crate::consts::{CACHED_PRINCIPAL_DEPTH, TMDB_WAIT_DEADLINE_SECS};
|
||||
use crate::tmdb_budget::{Lease, TmdbBudget};
|
||||
use crate::tmdb_client::{
|
||||
principal_top_n, TmdbCall, TmdbClient, TmdbExternalIds, TmdbHit, TmdbKind, TmdbTvDetails,
|
||||
};
|
||||
|
||||
/// TMDB resolution with a persistent cache in front and a token budget behind.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Resolver {
|
||||
pub(crate) client: Arc<TmdbClient>,
|
||||
pub(crate) cache: MidhashCache,
|
||||
pub(crate) budget: Arc<TmdbBudget>,
|
||||
}
|
||||
|
||||
impl Resolver {
|
||||
pub(crate) fn new(client: Arc<TmdbClient>, cache: MidhashCache, budget: Arc<TmdbBudget>) -> Self {
|
||||
Self {
|
||||
client,
|
||||
cache,
|
||||
budget,
|
||||
}
|
||||
}
|
||||
|
||||
/// Acquire a budget token, run `call`, map a hit through `on_hit`. A 429
|
||||
/// pauses the shared bucket globally rather than being retried here.
|
||||
async fn budgeted<T, R>(
|
||||
&self,
|
||||
call: impl std::future::Future<Output = TmdbCall<T>>,
|
||||
on_hit: impl FnOnce(T) -> R,
|
||||
) -> Option<R> {
|
||||
if matches!(
|
||||
self.budget
|
||||
.acquire(Duration::from_secs(TMDB_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
match call.await {
|
||||
TmdbCall::Hit(x) => Some(on_hit(x)),
|
||||
TmdbCall::Miss => None,
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
self.budget.note_429(retry);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached `GET /3/tv/{id}`. Entries written before the display/AKA fields
|
||||
/// existed self-heal with one refetch.
|
||||
pub(crate) async fn tv_details(&self, tmdbid: u64) -> Option<TmdbTvDetails> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_tvdetails(&key) {
|
||||
if let Ok(details) = serde_json::from_str::<TmdbTvDetails>(&json) {
|
||||
if details.has_display() && details.has_akas() {
|
||||
return Some(details);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.budgeted(self.client.tv_details(tmdbid), |details| {
|
||||
if let Ok(json) = serde_json::to_string(&details) {
|
||||
let _ = self.cache.put_tmdb_tvdetails(&key, &json);
|
||||
}
|
||||
details
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cached `GET /3/movie/{id}`.
|
||||
pub(crate) async fn movie_hit(&self, tmdbid: u64) -> Option<TmdbHit> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_moviedetails(&key) {
|
||||
if json != "null" {
|
||||
if let Ok(hit) = serde_json::from_str::<TmdbHit>(&json) {
|
||||
return Some(hit);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.budgeted(self.client.movie_details(tmdbid), |details| {
|
||||
let hit = details.into_hit();
|
||||
if let Ok(json) = serde_json::to_string(&hit) {
|
||||
let _ = self.cache.put_tmdb_moviedetails(&key, &json);
|
||||
}
|
||||
hit
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cached `external_ids` — the source of the card's `tvdbid` / `imdbid`.
|
||||
/// External ids are immutable, so a hit is cached forever.
|
||||
pub(crate) async fn external_ids(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
tmdbid: u64,
|
||||
) -> Option<TmdbExternalIds> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_extids(&key) {
|
||||
return serde_json::from_str(&json).ok();
|
||||
}
|
||||
self.budgeted(self.client.external_ids(kind, tmdbid), |ids| {
|
||||
if let Ok(json) = serde_json::to_string(&ids) {
|
||||
let _ = self.cache.put_tmdb_extids(&key, &json);
|
||||
}
|
||||
ids
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cached principal `search/multi`: free text → up to `n` confident
|
||||
/// `(tmdbid, kind)` candidates, most popular first.
|
||||
///
|
||||
/// The full ranked list (up to [`CACHED_PRINCIPAL_DEPTH`]) is persisted and
|
||||
/// sliced on read, so changing the top-N knob never needs a cache wipe.
|
||||
/// A transient TMDB failure is **not** negative-cached — the client folds
|
||||
/// "no results" and "timeout" into the same `Miss`, and caching the latter
|
||||
/// once left a keyword permanently card-less until the next cache wipe.
|
||||
pub(crate) async fn principal_top_n(&self, query: &str, n: usize) -> Vec<(u64, TmdbKind)> {
|
||||
let key = query.trim().to_lowercase();
|
||||
if key.is_empty() || n == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_principal_topn(&key) {
|
||||
let mut list = decode_principal_list(&json);
|
||||
list.truncate(n);
|
||||
return list;
|
||||
}
|
||||
if matches!(
|
||||
self.budget
|
||||
.acquire(Duration::from_secs(TMDB_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
return Vec::new(); // transient — don't poison the cache
|
||||
}
|
||||
let ranked: Vec<(u64, TmdbKind)> = match self.client.search_multi(query).await {
|
||||
TmdbCall::Hit(hits) => principal_top_n(&hits, query, CACHED_PRINCIPAL_DEPTH)
|
||||
.into_iter()
|
||||
.filter_map(|h| h.kind().map(|k| (h.id, k)))
|
||||
.collect(),
|
||||
TmdbCall::Miss => return Vec::new(),
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
self.budget.note_429(retry);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
// An empty list here is a genuine negative ("TMDB answered, nothing
|
||||
// passed the relevance gate") and is safe to remember.
|
||||
let _ = self
|
||||
.cache
|
||||
.put_tmdb_principal_topn(&key, &encode_principal_list(&ranked));
|
||||
let mut out = ranked;
|
||||
out.truncate(n);
|
||||
out
|
||||
}
|
||||
|
||||
/// Resolve one `(tmdbid, kind)` into a full [`Card`] — details plus the
|
||||
/// cross-source id bag. `None` when the by-id lookup fails.
|
||||
pub(crate) async fn card_by_id(&self, tmdbid: u64, kind: TmdbKind) -> Option<Card> {
|
||||
let ext = self.external_ids(kind, tmdbid).await.unwrap_or_default();
|
||||
match kind {
|
||||
TmdbKind::Tv => {
|
||||
let details = self.tv_details(tmdbid).await?;
|
||||
let year = details
|
||||
.first_air_date
|
||||
.as_deref()
|
||||
.and_then(|d| d.get(0..4))
|
||||
.and_then(|y| y.parse::<u16>().ok());
|
||||
Some(Card {
|
||||
source: CardSource::Tmdb,
|
||||
kind,
|
||||
source_id: format!("tv:{tmdbid}"),
|
||||
tmdbid: Some(tmdbid),
|
||||
tvdb_id: ext.tvdb_id,
|
||||
imdb_id: ext.imdb_id,
|
||||
title: details.name.clone(),
|
||||
guard_titles: Arc::new(dedup_titles(details.guard_titles())),
|
||||
overview: details.overview.clone().filter(|s| !s.is_empty()),
|
||||
poster_path: details.poster_path.clone().filter(|s| !s.is_empty()),
|
||||
year,
|
||||
seasons: Card::clamp_seasons(details.number_of_seasons),
|
||||
season_summaries: Arc::new(details.seasons.clone()),
|
||||
})
|
||||
}
|
||||
TmdbKind::Movie => {
|
||||
let hit = self.movie_hit(tmdbid).await?;
|
||||
let mut titles = vec![hit.title.clone()];
|
||||
if let Some(o) = &hit.original_title {
|
||||
titles.push(o.clone());
|
||||
}
|
||||
titles.extend(hit.alt_titles.iter().cloned());
|
||||
Some(Card {
|
||||
source: CardSource::Tmdb,
|
||||
kind,
|
||||
source_id: format!("movie:{tmdbid}"),
|
||||
tmdbid: Some(tmdbid),
|
||||
tvdb_id: None,
|
||||
imdb_id: ext.imdb_id,
|
||||
title: hit.title,
|
||||
guard_titles: Arc::new(dedup_titles(titles)),
|
||||
overview: hit.overview.filter(|s| !s.is_empty()),
|
||||
poster_path: hit.poster_path.filter(|s| !s.is_empty()),
|
||||
year: hit.year,
|
||||
seasons: 0,
|
||||
season_summaries: Arc::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Free text → up to `top_n` cards, most popular first.
|
||||
pub(crate) async fn cards_for_text(&self, free_text: &str, top_n: usize) -> Vec<Card> {
|
||||
let mut cards = Vec::new();
|
||||
for (id, kind) in self.principal_top_n(free_text, top_n).await {
|
||||
if let Some(card) = self.card_by_id(id, kind).await {
|
||||
cards.push(card);
|
||||
}
|
||||
}
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "tmdb",
|
||||
query = free_text,
|
||||
count = cards.len(),
|
||||
"resolved cards"
|
||||
);
|
||||
cards
|
||||
}
|
||||
}
|
||||
|
||||
/// Case-insensitive de-dup preserving first-seen order.
|
||||
fn dedup_titles(titles: Vec<String>) -> Vec<String> {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
titles
|
||||
.into_iter()
|
||||
.map(|t| t.trim().to_string())
|
||||
.filter(|t| !t.is_empty() && seen.insert(t.to_lowercase()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// JSON shape persisted in the principal-search cache. Kept byte-identical to
|
||||
/// the indexer feeder's so a warm cache is shared, not re-earned.
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
struct PrincipalEntry {
|
||||
tmdbid: u64,
|
||||
kind: String,
|
||||
}
|
||||
|
||||
fn kind_str(k: TmdbKind) -> &'static str {
|
||||
match k {
|
||||
TmdbKind::Movie => "movie",
|
||||
TmdbKind::Tv => "tv",
|
||||
}
|
||||
}
|
||||
|
||||
fn kind_from_str(s: &str) -> Option<TmdbKind> {
|
||||
match s {
|
||||
"movie" => Some(TmdbKind::Movie),
|
||||
"tv" => Some(TmdbKind::Tv),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_principal_list(list: &[(u64, TmdbKind)]) -> String {
|
||||
let entries: Vec<PrincipalEntry> = list
|
||||
.iter()
|
||||
.map(|&(tmdbid, kind)| PrincipalEntry {
|
||||
tmdbid,
|
||||
kind: kind_str(kind).to_string(),
|
||||
})
|
||||
.collect();
|
||||
serde_json::to_string(&entries).unwrap_or_else(|_| "[]".to_string())
|
||||
}
|
||||
|
||||
fn decode_principal_list(json: &str) -> Vec<(u64, TmdbKind)> {
|
||||
serde_json::from_str::<Vec<PrincipalEntry>>(json)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|e| kind_from_str(&e.kind).map(|k| (e.tmdbid, k)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The cache format must stay byte-compatible with the indexer feeder's —
|
||||
/// the two share redb table names, so a drift here silently invalidates a
|
||||
/// warm cache instead of failing loudly.
|
||||
#[test]
|
||||
fn principal_list_round_trips() {
|
||||
let list = vec![(95479, TmdbKind::Tv), (27205, TmdbKind::Movie)];
|
||||
let json = encode_principal_list(&list);
|
||||
assert_eq!(json, r#"[{"tmdbid":95479,"kind":"tv"},{"tmdbid":27205,"kind":"movie"}]"#);
|
||||
assert_eq!(decode_principal_list(&json), list);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn principal_list_tolerates_garbage_and_unknown_kinds() {
|
||||
assert!(decode_principal_list("not json").is_empty());
|
||||
assert!(decode_principal_list(r#"[{"tmdbid":1,"kind":"person"}]"#).is_empty());
|
||||
assert!(decode_principal_list("[]").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedup_titles_is_case_insensitive_and_order_preserving() {
|
||||
let out = dedup_titles(vec![
|
||||
"Frieren".to_string(),
|
||||
" frieren ".to_string(),
|
||||
"Sousou no Frieren".to_string(),
|
||||
"".to_string(),
|
||||
]);
|
||||
assert_eq!(out, vec!["Frieren", "Sousou no Frieren"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
//! The `tmdb` upstream — a **card** feeder plugin.
|
||||
//!
|
||||
//! Answers the discovery half of gateway search: free text in, `fileType=card`
|
||||
//! records out. It never calls an indexer and never serves bytes, which is the
|
||||
//! whole point — a card query costs at most one cached TMDB `search/multi`, so
|
||||
//! the expensive indexer budget is spent later, entirely on the single card the
|
||||
//! user clicks (design doc §1.2).
|
||||
//!
|
||||
//! When MyAnimeList lands it becomes a sibling `FeederPlugin` in this same
|
||||
//! binary, declaring the same `served_file_types = ["card"]`.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use meta_feeder_sdk::cache::MidhashCache;
|
||||
use meta_feeder_sdk::config::ConfigSchema;
|
||||
use meta_feeder_sdk::plugin::{
|
||||
ConfigError, FeederPlugin, GatewayQuery, HashKind, HashOutcome,
|
||||
};
|
||||
use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::card::{split_record_id, Card, CardSource};
|
||||
use crate::consts::DEFAULT_CARD_TOP_N;
|
||||
use crate::resolve::Resolver;
|
||||
use crate::tmdb_budget::{TmdbBudget, DEFAULT_TMDB_BURST, DEFAULT_TMDB_REFILL_PER_SEC};
|
||||
use crate::tmdb_client::TmdbClient;
|
||||
|
||||
/// Config the operator supplies via the feeder's own web form (invariant 12 —
|
||||
/// per-plugin config lives on the feeder, never on the gateway).
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct TmdbConfig {
|
||||
#[serde(default)]
|
||||
pub tmdb_token: String,
|
||||
#[serde(default)]
|
||||
pub tmdb_rate_per_sec: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub tmdb_burst: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub card_top_n: Option<usize>,
|
||||
}
|
||||
|
||||
impl TmdbConfig {
|
||||
/// Env seed, used on first boot before any `config.json` exists.
|
||||
pub fn from_env() -> Self {
|
||||
Self {
|
||||
tmdb_token: std::env::var("TMDB_TOKEN").unwrap_or_default(),
|
||||
tmdb_rate_per_sec: std::env::var("TMDB_RATE_PER_SEC")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok()),
|
||||
tmdb_burst: std::env::var("TMDB_BURST").ok().and_then(|v| v.parse().ok()),
|
||||
card_top_n: std::env::var("CARD_TOP_N").ok().and_then(|v| v.parse().ok()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TmdbCardPlugin {
|
||||
config: TmdbConfig,
|
||||
/// `None` until `configure` runs, and stays `None` when no TMDB token was
|
||||
/// supplied — the soft-skip path. The feeder still serves `/health` so
|
||||
/// `depends_on` is satisfied; queries just return nothing.
|
||||
resolver: Option<Resolver>,
|
||||
top_n: usize,
|
||||
/// Test hook: `(api_base, image_base)` overriding TMDB's real endpoints, so
|
||||
/// the contract test can point the client at a wiremock server.
|
||||
api_bases: Option<(String, String)>,
|
||||
}
|
||||
|
||||
impl Default for TmdbCardPlugin {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl TmdbCardPlugin {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
config: TmdbConfig::from_env(),
|
||||
resolver: None,
|
||||
top_n: DEFAULT_CARD_TOP_N,
|
||||
api_bases: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Test constructor: a fixed token plus TMDB endpoint overrides, so the
|
||||
/// contract test can drive the real router against a wiremock TMDB.
|
||||
pub fn with_api_base(token: &str, api_base: String, image_base: String) -> Self {
|
||||
let mut p = Self::new();
|
||||
p.config.tmdb_token = token.to_string();
|
||||
p.api_bases = Some((api_base, image_base));
|
||||
p
|
||||
}
|
||||
|
||||
/// File wins over env, exactly like every other feeder: `config.json` in the
|
||||
/// per-plugin cache dir is read at `configure` time, falling back to the env
|
||||
/// seed. No hot reload — bounce the feeder.
|
||||
fn load_config(&mut self, cache_dir: &Path) {
|
||||
let path = cache_dir.join("config.json");
|
||||
if let Ok(bytes) = std::fs::read(&path) {
|
||||
if let Ok(file_cfg) = serde_json::from_slice::<TmdbConfig>(&bytes) {
|
||||
if !file_cfg.tmdb_token.trim().is_empty() {
|
||||
self.config.tmdb_token = file_cfg.tmdb_token;
|
||||
}
|
||||
if file_cfg.tmdb_rate_per_sec.is_some() {
|
||||
self.config.tmdb_rate_per_sec = file_cfg.tmdb_rate_per_sec;
|
||||
}
|
||||
if file_cfg.tmdb_burst.is_some() {
|
||||
self.config.tmdb_burst = file_cfg.tmdb_burst;
|
||||
}
|
||||
if file_cfg.card_top_n.is_some() {
|
||||
self.config.card_top_n = file_cfg.card_top_n;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolver(&self) -> Option<&Resolver> {
|
||||
self.resolver.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FeederPlugin for TmdbCardPlugin {
|
||||
fn upstream_id(&self) -> &'static str {
|
||||
"tmdb"
|
||||
}
|
||||
|
||||
/// **The routing declaration.** `card` on the fileType axis is what makes
|
||||
/// meta-search fan a `fileType:card` query out to this gateway — no
|
||||
/// meta-search change was needed to add the tier (design doc §6). The
|
||||
/// content kinds are the *work* kinds a card can describe.
|
||||
fn served_file_types(&self) -> &'static [&'static str] {
|
||||
&["card"]
|
||||
}
|
||||
|
||||
fn served_content_kinds(&self) -> &'static [&'static str] {
|
||||
&["movie", "series"]
|
||||
}
|
||||
|
||||
fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
|
||||
self.load_config(cache_dir);
|
||||
let cache: MidhashCache = meta_feeder_sdk::common::open_midhash_cache(cache_dir, "tmdb")?;
|
||||
self.top_n = self.config.card_top_n.filter(|n| *n > 0).unwrap_or(DEFAULT_CARD_TOP_N);
|
||||
|
||||
let token = self.config.tmdb_token.trim().to_string();
|
||||
if token.is_empty() {
|
||||
// Soft-skip, matching every other opt-in upstream: stay healthy so
|
||||
// the gateway's `depends_on` is satisfied, but answer nothing.
|
||||
warn!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "tmdb",
|
||||
"no TMDB token configured; card queries will return nothing \
|
||||
(set it on the feeder's config page or via TMDB_TOKEN)"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
// `TmdbBudget::new` already returns an `Arc`.
|
||||
let budget = TmdbBudget::new(
|
||||
self.config
|
||||
.tmdb_rate_per_sec
|
||||
.filter(|v| *v > 0.0)
|
||||
.unwrap_or(DEFAULT_TMDB_REFILL_PER_SEC),
|
||||
self.config
|
||||
.tmdb_burst
|
||||
.filter(|v| *v > 0.0)
|
||||
.unwrap_or(DEFAULT_TMDB_BURST),
|
||||
);
|
||||
let client = match &self.api_bases {
|
||||
Some((api, image)) => TmdbClient::with_bases(token, api.clone(), image.clone()),
|
||||
None => TmdbClient::new(token),
|
||||
};
|
||||
self.resolver = Some(Resolver::new(Arc::new(client), cache, budget));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn health(&self) -> PluginHealth {
|
||||
PluginHealth::Ok
|
||||
}
|
||||
|
||||
async fn handle_query(
|
||||
&self,
|
||||
query: &GatewayQuery,
|
||||
max_results: usize,
|
||||
) -> Result<Vec<DiscoveryRecord>, GatewayError> {
|
||||
// Layer A early-return: this plugin only ever serves cards.
|
||||
if !meta_feeder_sdk::query_eval::query_accepts_plugin(
|
||||
query,
|
||||
self.served_file_types(),
|
||||
self.served_content_kinds(),
|
||||
) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let Some(resolver) = self.resolver() else {
|
||||
return Ok(Vec::new()); // soft-skipped (no token)
|
||||
};
|
||||
|
||||
// An explicit `tmdbid:` filter resolves exactly one card — the direct
|
||||
// "give me this work" lookup a deep link uses. Otherwise the free text
|
||||
// goes through the principal search.
|
||||
let cards: Vec<Card> = if let Some(id) = query
|
||||
.filters
|
||||
.get("tmdbid")
|
||||
.and_then(|v| v.first())
|
||||
.and_then(|s| s.trim().parse::<u64>().ok())
|
||||
{
|
||||
let kind = kind_hint(query, resolver, id).await;
|
||||
resolver.card_by_id(id, kind).await.into_iter().collect()
|
||||
} else {
|
||||
let free_text = query.free_text.trim();
|
||||
if free_text.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let n = self.top_n.min(max_results.max(1));
|
||||
resolver.cards_for_text(free_text, n).await
|
||||
};
|
||||
|
||||
let client = &resolver.client;
|
||||
Ok(cards
|
||||
.iter()
|
||||
.filter_map(|c| c.to_record(client, &query.filters))
|
||||
.take(max_results)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Resolve a card `record_id` back into its locator CID.
|
||||
///
|
||||
/// This is **not** a stub for a byte-less plugin — it is the card path. The
|
||||
/// outcome carries `bytes: None`, so the gateway core's three-branch
|
||||
/// auto-store routes it to `store_metadata_only`: the record lands in
|
||||
/// meta-core, nothing is written to WebDAV, and nothing is seeded to
|
||||
/// bitswap (seeding is gated on `Sha2_256`). The record IS the payload.
|
||||
///
|
||||
/// The CID needs no network call — it is a pure function of the record id,
|
||||
/// which is itself `"<source>:<source_id>"`. We re-resolve the card anyway
|
||||
/// so the stored record is complete; a resolution failure still yields the
|
||||
/// CID, because the identity is derivable regardless.
|
||||
async fn compute_outcomes(&self, record_id: &str) -> Result<Vec<HashOutcome>, GatewayError> {
|
||||
let (source, source_id) = split_record_id(record_id).ok_or_else(|| {
|
||||
GatewayError::Permanent(format!("malformed card record_id '{record_id}'"))
|
||||
})?;
|
||||
if source != CardSource::Tmdb.as_str() {
|
||||
return Err(GatewayError::NotFound);
|
||||
}
|
||||
let hash = meta_feeder_sdk::hash::compute_card_cid(source, source_id).ok_or_else(|| {
|
||||
GatewayError::Permanent(format!("card id too long to encode: '{record_id}'"))
|
||||
})?;
|
||||
|
||||
let record = match (self.resolver(), parse_source_id(source_id)) {
|
||||
(Some(resolver), Some((kind, tmdbid))) => resolver
|
||||
.card_by_id(tmdbid, kind)
|
||||
.await
|
||||
.and_then(|c| c.to_record(&resolver.client, &Default::default())),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(vec![HashOutcome {
|
||||
hash: Hash(hash),
|
||||
hash_kind: HashKind::CardLocator,
|
||||
bytes: None,
|
||||
record,
|
||||
file_extension: None,
|
||||
}])
|
||||
}
|
||||
|
||||
fn config_schema(&self) -> ConfigSchema {
|
||||
use meta_feeder_sdk::config::{ConfigField as F, ConfigSchema};
|
||||
ConfigSchema {
|
||||
fields: vec![
|
||||
F::secret("tmdb_token", "TMDB v4 token").with_help(
|
||||
"Read-access token from themoviedb.org. Without it this feeder \
|
||||
soft-skips and card searches return nothing.",
|
||||
),
|
||||
F::number("tmdb_rate_per_sec", "TMDB rate (req/s)").with_help(
|
||||
"Sustained TMDB request rate. Blank keeps the built-in default \
|
||||
(20/s). Takes effect on the next feeder restart.",
|
||||
),
|
||||
F::number("tmdb_burst", "TMDB burst (tokens)").with_help(
|
||||
"Short-burst TMDB token ceiling. Blank keeps the built-in \
|
||||
default (20). Takes effect on the next feeder restart.",
|
||||
),
|
||||
F::number("card_top_n", "Cards per search").with_help(
|
||||
"How many works a free-text search returns, most popular first. \
|
||||
Blank keeps the built-in default (10). Unlike the old anchor \
|
||||
top-N this costs no indexer requests — only TMDB.",
|
||||
),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn config_values(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"tmdb_token": self.config.tmdb_token,
|
||||
"tmdb_rate_per_sec": self.config.tmdb_rate_per_sec,
|
||||
"tmdb_burst": self.config.tmdb_burst,
|
||||
"card_top_n": self.config.card_top_n,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide whether an explicit `tmdbid:` refers to a show or a film: an explicit
|
||||
/// `contentKind` filter wins, else probe TV details (a movie id has none).
|
||||
async fn kind_hint(query: &GatewayQuery, resolver: &Resolver, id: u64) -> crate::tmdb_client::TmdbKind {
|
||||
use crate::tmdb_client::TmdbKind;
|
||||
match query
|
||||
.filters
|
||||
.get("contentKind")
|
||||
.and_then(|v| v.first())
|
||||
.map(|s| s.as_str())
|
||||
{
|
||||
Some("movie") => return TmdbKind::Movie,
|
||||
Some("series") | Some("episode") | Some("tvshow") => return TmdbKind::Tv,
|
||||
_ => {}
|
||||
}
|
||||
match resolver.tv_details(id).await {
|
||||
Some(d) if d.number_of_seasons > 0 => TmdbKind::Tv,
|
||||
_ => TmdbKind::Movie,
|
||||
}
|
||||
}
|
||||
|
||||
/// `"tv:95479"` / `"movie:27205"` → `(kind, tmdbid)`.
|
||||
fn parse_source_id(source_id: &str) -> Option<(crate::tmdb_client::TmdbKind, u64)> {
|
||||
use crate::tmdb_client::TmdbKind;
|
||||
let (kind, id) = source_id.split_once(':')?;
|
||||
let kind = match kind {
|
||||
"tv" => TmdbKind::Tv,
|
||||
"movie" => TmdbKind::Movie,
|
||||
_ => return None,
|
||||
};
|
||||
Some((kind, id.trim().parse().ok()?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::tmdb_client::TmdbKind;
|
||||
|
||||
#[test]
|
||||
fn source_id_parses_both_kinds() {
|
||||
assert!(matches!(
|
||||
parse_source_id("tv:95479"),
|
||||
Some((TmdbKind::Tv, 95479))
|
||||
));
|
||||
assert!(matches!(
|
||||
parse_source_id("movie:27205"),
|
||||
Some((TmdbKind::Movie, 27205))
|
||||
));
|
||||
assert!(parse_source_id("person:123").is_none());
|
||||
assert!(parse_source_id("tv:notanumber").is_none());
|
||||
assert!(parse_source_id("95479").is_none());
|
||||
}
|
||||
|
||||
/// The routing declaration is load-bearing: `card` here is the only reason
|
||||
/// meta-search fans a `fileType:card` query out to this gateway.
|
||||
#[test]
|
||||
fn declares_the_card_file_type() {
|
||||
let p = TmdbCardPlugin::new();
|
||||
assert_eq!(p.served_file_types(), &["card"]);
|
||||
assert_eq!(p.served_content_kinds(), &["movie", "series"]);
|
||||
assert_eq!(p.upstream_id(), "tmdb");
|
||||
}
|
||||
|
||||
/// A card CID is derivable with no network and no resolver, so
|
||||
/// `compute_outcomes` must still produce one when the feeder is
|
||||
/// soft-skipped — only the record is missing.
|
||||
#[tokio::test]
|
||||
async fn compute_outcomes_yields_a_byteless_locator_without_a_token() {
|
||||
let p = TmdbCardPlugin::new();
|
||||
let outcomes = p.compute_outcomes("tmdb:tv:95479").await.expect("outcomes");
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
let o = &outcomes[0];
|
||||
assert_eq!(o.hash_kind, HashKind::CardLocator);
|
||||
assert!(o.bytes.is_none(), "a card has no bytes, ever");
|
||||
assert!(o.file_extension.is_none());
|
||||
assert_eq!(
|
||||
o.hash.0,
|
||||
meta_feeder_sdk::hash::compute_card_cid("tmdb", "tv:95479").unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compute_outcomes_rejects_a_foreign_or_malformed_id() {
|
||||
let p = TmdbCardPlugin::new();
|
||||
assert!(p.compute_outcomes("mal:52991").await.is_err());
|
||||
assert!(p.compute_outcomes("nosource").await.is_err());
|
||||
}
|
||||
|
||||
/// Soft-skip: no token means no cards, not a hard failure — the feeder
|
||||
/// stays healthy so the gateway's `depends_on` is satisfied.
|
||||
#[tokio::test]
|
||||
async fn queries_return_nothing_without_a_token() {
|
||||
let p = TmdbCardPlugin::new();
|
||||
let q = GatewayQuery::from_free_text("frieren");
|
||||
assert!(p.handle_query(&q, 10).await.unwrap().is_empty());
|
||||
assert!(matches!(p.health(), PluginHealth::Ok));
|
||||
}
|
||||
|
||||
/// Layer-A routing: a query for something this plugin cannot serve returns
|
||||
/// immediately without touching TMDB.
|
||||
#[tokio::test]
|
||||
async fn rejects_a_query_for_a_type_it_does_not_serve() {
|
||||
let p = TmdbCardPlugin::new();
|
||||
let mut q = GatewayQuery::from_free_text("frieren");
|
||||
q.filters
|
||||
.insert("fileType".to_string(), vec!["video".to_string()]);
|
||||
assert!(p.handle_query(&q, 10).await.unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
//! Process-global TMDB request budget.
|
||||
//!
|
||||
//! TMDB's free tier caps at ~40 requests / 10 s. The gateway can field many
|
||||
//! concurrent searches, each wanting to enrich several records, so without a
|
||||
//! shared throttle a burst of discover queries collectively blows the limit
|
||||
//! and TMDB starts returning 429s — which surfaces as empty home rows in
|
||||
//! meta-watch. This is a single hand-rolled token bucket shared by every
|
||||
//! enrichment task in the process (one `Arc<TmdbBudget>` injected into the
|
||||
//! torznab plugin via [`meta_feeder_sdk::plugin::FeederPlugin::set_tmdb_budget`]).
|
||||
//!
|
||||
//! Design notes:
|
||||
//! - **Lazy refill.** Tokens accrue on each `acquire` from elapsed wall-clock,
|
||||
//! so there is no background timer task and no idle cost.
|
||||
//! - **Cancel-safe.** The token is decremented synchronously under the lock
|
||||
//! right before `acquire` returns [`Lease::Granted`]; there is no `.await`
|
||||
//! between the decision to grant and the return, so a dropped `acquire`
|
||||
//! future (consumer disconnected) never spends a token.
|
||||
//! - **Global 429 pause.** A single upstream 429 freezes *all* grants until
|
||||
//! the Retry-After window elapses via [`TmdbBudget::note_429`].
|
||||
//!
|
||||
//! The critical sections hold a plain `std::sync::Mutex` (no awaits inside),
|
||||
//! while the waiting happens outside the lock on a `tokio::time::sleep` raced
|
||||
//! against a `tokio::sync::Notify`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
/// Default sustained TMDB grant rate (requests/second). TMDB removed its old
|
||||
/// 40-req/10-s hard limit years ago (now effectively ~50 req/s, "be reasonable"),
|
||||
/// and the `tmdb_inflight` single-flight dedup already collapses same-show lookups
|
||||
/// to one search — so a conservative 20/s is well under the ceiling while removing
|
||||
/// the enrichment serializer that made the first poster land ~25 s late. Operator-
|
||||
/// overridable via the `tmdb_rate_per_sec` feeder web-config field.
|
||||
pub const DEFAULT_TMDB_REFILL_PER_SEC: f64 = 20.0;
|
||||
/// Default TMDB burst ceiling (tokens) — a short burst of concurrent searches
|
||||
/// is served immediately, then throttled to the sustained rate. Overridable via
|
||||
/// the `tmdb_burst` feeder web-config field.
|
||||
pub const DEFAULT_TMDB_BURST: f64 = 20.0;
|
||||
|
||||
/// Outcome of an [`TmdbBudget::acquire`] call.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Lease {
|
||||
/// A token was granted; the caller may make one TMDB API call.
|
||||
Granted,
|
||||
/// The wait deadline elapsed before a token became available. The caller
|
||||
/// should proceed without enrichment (best-effort degrade).
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
struct BucketState {
|
||||
tokens: f64,
|
||||
capacity: f64,
|
||||
refill_per_sec: f64,
|
||||
last_refill: Instant,
|
||||
/// When set and in the future, all grants are frozen (global 429 pause).
|
||||
paused_until: Option<Instant>,
|
||||
}
|
||||
|
||||
impl BucketState {
|
||||
/// Accrue tokens for elapsed time since the last refill. Called under the
|
||||
/// lock before inspecting `tokens`.
|
||||
fn refill(&mut self, now: Instant) {
|
||||
let elapsed = now
|
||||
.saturating_duration_since(self.last_refill)
|
||||
.as_secs_f64();
|
||||
if elapsed > 0.0 {
|
||||
self.tokens = (self.tokens + elapsed * self.refill_per_sec).min(self.capacity);
|
||||
self.last_refill = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared token bucket gating TMDB API calls across the whole process.
|
||||
pub struct TmdbBudget {
|
||||
state: Mutex<BucketState>,
|
||||
/// Woken whenever a grant *might* now be possible (a token was returned to
|
||||
/// the bucket, or a 429 pause was cleared). Waiters re-check under the
|
||||
/// lock after waking; a missed wake only costs an extra bounded sleep.
|
||||
wake: Notify,
|
||||
}
|
||||
|
||||
impl TmdbBudget {
|
||||
/// Build a budget that sustains `refill_per_sec` grants/second with a burst
|
||||
/// ceiling of `capacity` tokens. The bucket starts full so the first burst
|
||||
/// is served immediately.
|
||||
pub fn new(refill_per_sec: f64, capacity: f64) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(BucketState {
|
||||
tokens: capacity,
|
||||
capacity,
|
||||
refill_per_sec,
|
||||
last_refill: Instant::now(),
|
||||
paused_until: None,
|
||||
}),
|
||||
wake: Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for one token, giving up after `deadline`. Cancel-safe: dropping
|
||||
/// the returned future before it resolves [`Lease::Granted`] spends no
|
||||
/// token.
|
||||
pub async fn acquire(&self, deadline: Duration) -> Lease {
|
||||
let give_up_at = Instant::now() + deadline;
|
||||
loop {
|
||||
// Decide-or-compute-wait, all under the lock (no awaits inside).
|
||||
let sleep_for = {
|
||||
let mut st = self.state.lock().expect("tmdb budget mutex poisoned");
|
||||
let now = Instant::now();
|
||||
if let Some(until) = st.paused_until {
|
||||
if now >= until {
|
||||
st.paused_until = None;
|
||||
}
|
||||
}
|
||||
let paused = st.paused_until;
|
||||
if paused.is_none() {
|
||||
st.refill(now);
|
||||
if st.tokens >= 1.0 {
|
||||
// Atomic grant: decrement and return with no await between.
|
||||
st.tokens -= 1.0;
|
||||
return Lease::Granted;
|
||||
}
|
||||
}
|
||||
if now >= give_up_at {
|
||||
return Lease::DeadlineExceeded;
|
||||
}
|
||||
let until_deadline = give_up_at.saturating_duration_since(now);
|
||||
let wait = match paused {
|
||||
// Frozen: wake when the pause is scheduled to lift.
|
||||
Some(until) => until.saturating_duration_since(now),
|
||||
// Throttled: wake when the next whole token should accrue.
|
||||
None => {
|
||||
let needed = 1.0 - st.tokens;
|
||||
let secs = if st.refill_per_sec > 0.0 {
|
||||
needed / st.refill_per_sec
|
||||
} else {
|
||||
until_deadline.as_secs_f64()
|
||||
};
|
||||
Duration::from_secs_f64(secs.max(0.0))
|
||||
}
|
||||
};
|
||||
wait.min(until_deadline)
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(sleep_for) => {}
|
||||
_ = self.wake.notified() => {}
|
||||
}
|
||||
if Instant::now() >= give_up_at {
|
||||
return Lease::DeadlineExceeded;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Feed back an upstream 429: pause all grants until `now + retry_after`.
|
||||
/// Extends an existing pause but never shortens it.
|
||||
pub fn note_429(&self, retry_after: Duration) {
|
||||
let until = Instant::now() + retry_after;
|
||||
{
|
||||
let mut st = self.state.lock().expect("tmdb budget mutex poisoned");
|
||||
st.paused_until = Some(match st.paused_until {
|
||||
Some(existing) if existing > until => existing,
|
||||
_ => until,
|
||||
});
|
||||
}
|
||||
// Wake waiters so they recompute their sleep against the new pause.
|
||||
self.wake.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-indexer registry of [`TmdbBudget`] token buckets.
|
||||
///
|
||||
/// Replaces the single global indexer limiter: each indexer is rate-limited
|
||||
/// independently, so a fast Usenet indexer isn't throttled to a slow torrent
|
||||
/// tracker's cadence, and N indexers fetched concurrently give ~N× aggregate
|
||||
/// throughput. A 429/reset on one indexer (via [`TmdbBudget::note_429`] on its
|
||||
/// own bucket) pauses only that indexer. Buckets are created lazily, keyed by
|
||||
/// `IndexerConfig.name` (unique per tracker after `effective_indexers`
|
||||
/// expansion). Shared by `Arc` across every concurrent search's plugin clone, so
|
||||
/// the per-indexer rate is real — all in-flight searches contend on the same
|
||||
/// bucket for a given indexer.
|
||||
pub struct IndexerLimiterRegistry {
|
||||
inner: Mutex<RegistryInner>,
|
||||
}
|
||||
|
||||
struct RegistryInner {
|
||||
buckets: HashMap<String, Arc<TmdbBudget>>,
|
||||
default_rps: f64,
|
||||
default_burst: f64,
|
||||
}
|
||||
|
||||
impl IndexerLimiterRegistry {
|
||||
/// Build a registry that seeds each lazily-created bucket at
|
||||
/// `default_rps` / `default_burst`.
|
||||
pub fn new(default_rps: f64, default_burst: f64) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
inner: Mutex::new(RegistryInner {
|
||||
buckets: HashMap::new(),
|
||||
default_rps,
|
||||
default_burst,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Override the default per-indexer rate (the `indexer_rate_per_sec`
|
||||
/// web-config field). Applied at `configure()` time — before any bucket is
|
||||
/// created — and clears existing buckets so a reconfigure re-seeds at the new
|
||||
/// rate. Non-positive `rps` is ignored (keeps the compiled default).
|
||||
pub fn set_default_rps(&self, rps: f64) {
|
||||
if rps > 0.0 {
|
||||
let mut inner = self.inner.lock().expect("indexer limiter registry poisoned");
|
||||
inner.default_rps = rps;
|
||||
inner.buckets.clear();
|
||||
}
|
||||
}
|
||||
|
||||
/// The token bucket for `indexer_name`, creating it on first use.
|
||||
pub fn for_indexer(&self, indexer_name: &str) -> Arc<TmdbBudget> {
|
||||
let mut inner = self.inner.lock().expect("indexer limiter registry poisoned");
|
||||
if let Some(bucket) = inner.buckets.get(indexer_name) {
|
||||
return bucket.clone();
|
||||
}
|
||||
let bucket = TmdbBudget::new(inner.default_rps, inner.default_burst);
|
||||
inner
|
||||
.buckets
|
||||
.insert(indexer_name.to_string(), bucket.clone());
|
||||
bucket
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn grants_up_to_capacity_immediately() {
|
||||
let b = TmdbBudget::new(3.0, 3.0);
|
||||
// Three tokens available at t0 — all granted without waiting.
|
||||
for _ in 0..3 {
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn fourth_token_waits_for_refill() {
|
||||
let b = TmdbBudget::new(2.0, 2.0); // 2/s sustained
|
||||
for _ in 0..2 {
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
}
|
||||
// Bucket empty; next grant must wait ~0.5s for one token at 2/s.
|
||||
let start = Instant::now();
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
let waited = start.elapsed();
|
||||
assert!(
|
||||
waited >= Duration::from_millis(450),
|
||||
"expected to wait ~0.5s for refill, waited {waited:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn deadline_exceeded_when_drained() {
|
||||
let b = TmdbBudget::new(0.001, 1.0); // effectively no refill within the test
|
||||
assert_eq!(b.acquire(Duration::from_secs(1)).await, Lease::Granted);
|
||||
// Bucket empty, refill far slower than the deadline.
|
||||
assert_eq!(
|
||||
b.acquire(Duration::from_millis(200)).await,
|
||||
Lease::DeadlineExceeded
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn note_429_pauses_grants_globally() {
|
||||
let b = TmdbBudget::new(100.0, 100.0); // plenty of tokens
|
||||
b.note_429(Duration::from_secs(2));
|
||||
// Even with tokens available, grants are frozen during the pause —
|
||||
// a short deadline times out.
|
||||
assert_eq!(
|
||||
b.acquire(Duration::from_millis(500)).await,
|
||||
Lease::DeadlineExceeded
|
||||
);
|
||||
// After the pause window, grants resume.
|
||||
assert_eq!(b.acquire(Duration::from_secs(5)).await, Lease::Granted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_lazily_keys_by_indexer_name() {
|
||||
let reg = IndexerLimiterRegistry::new(1.0, 3.0);
|
||||
let a1 = reg.for_indexer("nzb.life");
|
||||
let a2 = reg.for_indexer("nzb.life");
|
||||
let b = reg.for_indexer("nyaa.si");
|
||||
// Same name → same bucket instance; different name → distinct bucket.
|
||||
assert!(Arc::ptr_eq(&a1, &a2));
|
||||
assert!(!Arc::ptr_eq(&a1, &b));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn per_indexer_429_does_not_pause_other_indexers() {
|
||||
let reg = IndexerLimiterRegistry::new(1.0, 3.0);
|
||||
let hot = reg.for_indexer("hot");
|
||||
let cold = reg.for_indexer("cold");
|
||||
// A 429 on `hot` pauses only `hot`.
|
||||
hot.note_429(Duration::from_secs(5));
|
||||
assert_eq!(
|
||||
hot.acquire(Duration::from_millis(200)).await,
|
||||
Lease::DeadlineExceeded
|
||||
);
|
||||
// `cold` is untouched — its bucket starts full and grants immediately.
|
||||
assert_eq!(
|
||||
cold.acquire(Duration::from_millis(200)).await,
|
||||
Lease::Granted
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_set_default_rps_affects_new_buckets_only() {
|
||||
let reg = IndexerLimiterRegistry::new(1.0, 1.0);
|
||||
let _early = reg.for_indexer("early"); // created at 1.0
|
||||
reg.set_default_rps(50.0); // clears buckets; new default 50/s
|
||||
// Non-positive is ignored (keeps 50).
|
||||
reg.set_default_rps(0.0);
|
||||
// A freshly created bucket uses the updated default (burst 1 → one
|
||||
// immediate grant, then throttled at 50/s).
|
||||
let b = reg.for_indexer("late");
|
||||
// Smoke: the bucket exists and is usable (full-bucket first grant).
|
||||
assert!(Arc::ptr_eq(&b, ®.for_indexer("late")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,978 @@
|
||||
//! TMDB HTTP client + DTOs + season/episode bounds check.
|
||||
//!
|
||||
//! **Verbatim lift** of the indexer feeder's `tmdb.rs` (which was itself split
|
||||
//! out of the monolithic `torznab.rs`). Kept byte-compatible with that copy on
|
||||
//! purpose: TMDB's API handling is fiddly — 429/`Retry-After` semantics, the
|
||||
//! self-healing cache-shape checks, the AKA append — and a fix on either side
|
||||
//! should port across as a straight copy. The redb cache table names match for
|
||||
//! the same reason, so a warm cache is shared rather than re-earned.
|
||||
//!
|
||||
//! Consequence: this file carries surface the **card feeder does not use**.
|
||||
//! `search` / `discovery_list` (fuzzy title search, home rows),
|
||||
//! `season_episode_bounds` / `episode_in_season` (release season/episode
|
||||
//! reconciliation) and the language helpers all belong to *retrieval*, which
|
||||
//! stays in the indexer feeder. They are dead here by design, not by neglect —
|
||||
//! hence the module-wide allow rather than a trim that would fork the file.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use meta_feeder_sdk::common::urlencode;
|
||||
use std::time::Duration;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
// -- TMDB metadata enrichment ------------------------------------------------
|
||||
//
|
||||
// TMDB v4 bearer-token client. Two endpoints used:
|
||||
//
|
||||
// GET /3/search/{movie,tv}?query=<urlencoded>&include_adult=false[&year=<YYYY>]
|
||||
// GET <image_base>/<poster_path>
|
||||
//
|
||||
// The token-bearing `Authorization` header gates both. Free-tier rate
|
||||
// limit is ~40 req / 10 s — we don't hit it from a dev box but it's
|
||||
// worth knowing. All TMDB failures degrade silently: enrichment is a
|
||||
// best-effort augmentation, never a hard dep.
|
||||
|
||||
/// Lightweight TMDB v4 client. Cheap to clone (the underlying
|
||||
/// `reqwest::Client` is `Arc`-backed).
|
||||
#[derive(Clone)]
|
||||
pub struct TmdbClient {
|
||||
http: reqwest::Client,
|
||||
bearer_token: String,
|
||||
api_base: String,
|
||||
image_base: String,
|
||||
}
|
||||
|
||||
impl TmdbClient {
|
||||
pub fn new(bearer_token: String) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(TMDB_POSTER_TIMEOUT_SECS))
|
||||
.user_agent(USER_AGENT)
|
||||
.build()
|
||||
.expect("rustls reqwest client build infallible");
|
||||
Self {
|
||||
http,
|
||||
bearer_token,
|
||||
api_base: TMDB_API_BASE.to_string(),
|
||||
image_base: TMDB_IMAGE_BASE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// **The one deviation from the verbatim lift**: the indexer feeder gates
|
||||
/// this on `#[cfg(test)]`. The card feeder's contract test is an
|
||||
/// *integration* test, which compiles the lib without `cfg(test)`, so it
|
||||
/// needs a real constructor to point the client at a wiremock TMDB.
|
||||
pub fn with_bases(token: String, api_base: String, image_base: String) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(TMDB_POSTER_TIMEOUT_SECS))
|
||||
.user_agent(USER_AGENT)
|
||||
.build()
|
||||
.expect("rustls reqwest client build infallible");
|
||||
Self {
|
||||
http,
|
||||
bearer_token: token,
|
||||
api_base,
|
||||
image_base,
|
||||
}
|
||||
}
|
||||
|
||||
/// Search TMDB for a movie/TV title. Returns [`TmdbCall::Hit`] with the
|
||||
/// top result, [`TmdbCall::Miss`] when there are no matches (or any
|
||||
/// transient/timeout error — best-effort degrade), or
|
||||
/// [`TmdbCall::RateLimited`] carrying the Retry-After window on a 429 so
|
||||
/// the caller can pause the shared budget globally. Wraps the whole call
|
||||
/// (including JSON decode) in [`TMDB_SEARCH_TIMEOUT_SECS`].
|
||||
pub(crate) async fn search(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
title: &str,
|
||||
year: Option<u16>,
|
||||
) -> TmdbCall<TmdbHit> {
|
||||
let endpoint = match kind {
|
||||
TmdbKind::Movie => "search/movie",
|
||||
TmdbKind::Tv => "search/tv",
|
||||
};
|
||||
let mut url = format!(
|
||||
"{}/{}?query={}&include_adult=false",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
endpoint,
|
||||
urlencode(title),
|
||||
);
|
||||
if let Some(y) = year {
|
||||
// Movie uses `year`, TV uses `first_air_date_year`; both
|
||||
// are documented but movies accept `year` for backward-
|
||||
// compat which is what we want.
|
||||
url.push_str(&format!("&year={y}"));
|
||||
}
|
||||
let fut = self
|
||||
.http
|
||||
.get(&url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb search non-2xx; degrading"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
let body: TmdbSearchResponse =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json())
|
||||
.await
|
||||
{
|
||||
Ok(Ok(b)) => b,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
match body.results.into_iter().next() {
|
||||
Some(top) => TmdbCall::Hit(top.into_hit(kind)),
|
||||
None => TmdbCall::Miss,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the public TMDB poster CDN URL for `poster_path` (no fetch). In the
|
||||
/// feeder model the feeder no longer fetches+stores the poster bytes; it
|
||||
/// emits this URL as a `poster_url` field and the **gateway core** seeds it
|
||||
/// into a content-addressed `poster` cid (same path as giphy/wikicommons
|
||||
/// previews). Inserts the required [`TMDB_POSTER_SIZE`] segment.
|
||||
pub(crate) fn poster_cdn_url(&self, poster_path: &str) -> String {
|
||||
format!(
|
||||
"{}/{}/{}",
|
||||
self.image_base.trim_end_matches('/'),
|
||||
TMDB_POSTER_SIZE,
|
||||
poster_path.trim_start_matches('/'),
|
||||
)
|
||||
}
|
||||
|
||||
/// Fetch a poster image's raw bytes. `poster_path` is the
|
||||
/// TMDB-supplied path (typically `"/abcdef.jpg"` with leading slash).
|
||||
///
|
||||
/// `image_base` is the bare CDN root (`https://image.tmdb.org/t/p`); TMDB
|
||||
/// requires a **size segment** (`/w500`, `/original`, …) between the root
|
||||
/// and the file or it 404s. The const comment documents this contract and
|
||||
/// this caller honors it by inserting [`TMDB_POSTER_SIZE`].
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) async fn fetch_poster(&self, poster_path: &str) -> Option<bytes::Bytes> {
|
||||
let url = format!(
|
||||
"{}/{}/{}",
|
||||
self.image_base.trim_end_matches('/'),
|
||||
TMDB_POSTER_SIZE,
|
||||
poster_path.trim_start_matches('/'),
|
||||
);
|
||||
let resp = self.http.get(&url).send().await.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
poster_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb poster fetch non-2xx; skipping"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
resp.bytes().await.ok()
|
||||
}
|
||||
|
||||
/// Fetch a TV show's authoritative structure (`number_of_seasons`
|
||||
/// plus the per-season `episode_count`s) via `GET /3/tv/{id}`. Used
|
||||
/// only to bounds-check a title-parsed season/episode — see
|
||||
/// [`season_episode_bounds`]. Wrapped in the same search timeout;
|
||||
/// any failure returns `None` so validation degrades to "accept".
|
||||
pub(crate) async fn tv_details(&self, tmdbid: u64) -> TmdbCall<TmdbTvDetails> {
|
||||
let url = format!(
|
||||
"{}/tv/{}?append_to_response=alternative_titles",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
tmdbid,
|
||||
);
|
||||
let fut = self
|
||||
.http
|
||||
.get(&url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb tv-details non-2xx; skipping season/episode validation"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json()).await
|
||||
{
|
||||
Ok(Ok(d)) => TmdbCall::Hit(d),
|
||||
_ => TmdbCall::Miss,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch cross-database ids (`tvdb_id`, `imdb_id`) for a known tmdbid via
|
||||
/// `GET /3/{tv,movie}/{id}/external_ids`. The anchored torznab path needs
|
||||
/// `tvdb_id` to query indexers by `tvsearch&tvdbid=` (TMDB's id is not a
|
||||
/// standard `tvsearch` param). Same timeout/429 contract as [`tv_details`].
|
||||
pub(crate) async fn external_ids(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
tmdbid: u64,
|
||||
) -> TmdbCall<TmdbExternalIds> {
|
||||
let seg = match kind {
|
||||
TmdbKind::Movie => "movie",
|
||||
TmdbKind::Tv => "tv",
|
||||
};
|
||||
let url = format!(
|
||||
"{}/{}/{}/external_ids",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
seg,
|
||||
tmdbid,
|
||||
);
|
||||
self.get_json(&url, "external_ids").await
|
||||
}
|
||||
|
||||
/// Fetch a movie's canonical details via `GET /3/movie/{id}` (the anchored
|
||||
/// enrichment source for a known movie tmdbid — resolves title/overview/
|
||||
/// poster/year/imdb_id without a fuzzy title search). Same contract as
|
||||
/// [`tv_details`].
|
||||
pub(crate) async fn movie_details(&self, tmdbid: u64) -> TmdbCall<TmdbMovieDetails> {
|
||||
let url = format!(
|
||||
"{}/movie/{}?append_to_response=alternative_titles",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
tmdbid
|
||||
);
|
||||
self.get_json(&url, "movie-details").await
|
||||
}
|
||||
|
||||
/// Principal search via `GET /3/search/multi` — maps a bare keyword to a
|
||||
/// canonical tmdbid + media type so the gateway can then query indexers
|
||||
/// structurally (the "TMDB as the front door" identification step
|
||||
/// Sonarr/Radarr use). Returns the mixed movie/tv/person result list; the
|
||||
/// caller picks the top-N confident anchors via [`principal_top_n`].
|
||||
pub(crate) async fn search_multi(&self, query: &str) -> TmdbCall<Vec<TmdbMultiItem>> {
|
||||
let url = format!(
|
||||
"{}/search/multi?query={}&include_adult=false",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
urlencode(query),
|
||||
);
|
||||
match self
|
||||
.get_json::<TmdbMultiResponse>(&url, "search/multi")
|
||||
.await
|
||||
{
|
||||
TmdbCall::Hit(r) => TmdbCall::Hit(r.results),
|
||||
TmdbCall::Miss => TmdbCall::Miss,
|
||||
TmdbCall::RateLimited(d) => TmdbCall::RateLimited(d),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a TMDB **discovery list** (trending / popular / top-rated / a
|
||||
/// `/discover` query) and map each result to a [`TmdbHit`]. Powers the
|
||||
/// keyword-less discovery branch (see [`super::discovery`]): the caller
|
||||
/// composes `path_and_query` (e.g. `"movie/popular"`,
|
||||
/// `"trending/tv/week"`, or `"discover/tv?with_keywords=210024&sort_by=…"`)
|
||||
/// and passes the `kind` so movie-vs-TV title/date fields decode correctly.
|
||||
/// All these endpoints share the `{results: [TmdbSearchItem]}` shape, so one
|
||||
/// method covers them. Returns the page's hits, or `Miss`/`RateLimited` under
|
||||
/// the usual best-effort contract.
|
||||
pub(crate) async fn discovery_list(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
path_and_query: &str,
|
||||
) -> TmdbCall<Vec<TmdbHit>> {
|
||||
let url = format!(
|
||||
"{}/{}",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
path_and_query.trim_start_matches('/'),
|
||||
);
|
||||
match self.get_json::<TmdbSearchResponse>(&url, "discovery").await {
|
||||
TmdbCall::Hit(r) => {
|
||||
TmdbCall::Hit(r.results.into_iter().map(|i| i.into_hit(kind)).collect())
|
||||
}
|
||||
TmdbCall::Miss => TmdbCall::Miss,
|
||||
TmdbCall::RateLimited(d) => TmdbCall::RateLimited(d),
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared `GET <url>` → JSON helper with the standard bearer auth, search
|
||||
/// timeout, 429→`RateLimited`, non-2xx/transient→`Miss` contract. Factored
|
||||
/// out of [`tv_details`]/[`external_ids`]/[`movie_details`] (identical
|
||||
/// modulo the decoded type and the log label).
|
||||
async fn get_json<T: serde::de::DeserializeOwned>(&self, url: &str, what: &str) -> TmdbCall<T> {
|
||||
let fut = self
|
||||
.http
|
||||
.get(url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
// Transport error / timeout. Logged at debug so the otherwise
|
||||
// silent best-effort degrade is diagnosable.
|
||||
Ok(Err(e)) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, error=%e, "tmdb call send error; degrading");
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, "tmdb call send timeout; degrading");
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
what,
|
||||
"tmdb call non-2xx; degrading"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json()).await {
|
||||
Ok(Ok(d)) => TmdbCall::Hit(d),
|
||||
// A decode failure here folds into `Miss` (best-effort), but log it
|
||||
// at debug: a single malformed element fails the whole `Vec` decode,
|
||||
// which would otherwise silently disable e.g. multi-anchor.
|
||||
Ok(Err(e)) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, error=%e, "tmdb call decode error; degrading");
|
||||
TmdbCall::Miss
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, "tmdb call decode timeout; degrading");
|
||||
TmdbCall::Miss
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a TMDB API call. Distinguishes a 429 rate-limit (which feeds
|
||||
/// `Retry-After` into the shared [`TmdbBudget`] so all enrichment pauses
|
||||
/// globally) from a plain miss or transient error (best-effort degrade).
|
||||
pub(crate) enum TmdbCall<T> {
|
||||
Hit(T),
|
||||
Miss,
|
||||
RateLimited(Duration),
|
||||
}
|
||||
|
||||
/// Parse a TMDB `Retry-After` header into a pause duration, defaulting to
|
||||
/// [`TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS`] when absent/unparseable. Mirrors
|
||||
/// the indexer-path logic in [`map_status`].
|
||||
pub(crate) fn parse_tmdb_retry_after(resp: &reqwest::Response) -> Duration {
|
||||
let secs = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS);
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TmdbKind {
|
||||
Movie,
|
||||
Tv,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbSearchResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) results: Vec<TmdbSearchItem>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbSearchItem {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) original_name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) release_date: Option<String>, // movie, YYYY-MM-DD
|
||||
#[serde(default)]
|
||||
pub(crate) first_air_date: Option<String>, // tv, YYYY-MM-DD
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) genre_ids: Vec<u32>,
|
||||
/// ISO 639-1 (2-letter) language the title was originally produced in.
|
||||
/// Mapped to `lang3` and used to file `original_title` under
|
||||
/// `titles/{lang3}` (METADATA_KEYS.md §3).
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
}
|
||||
|
||||
impl TmdbSearchItem {
|
||||
pub(crate) fn into_hit(self, kind: TmdbKind) -> TmdbHit {
|
||||
let title = match kind {
|
||||
TmdbKind::Movie => self.title,
|
||||
TmdbKind::Tv => self.name,
|
||||
}
|
||||
.unwrap_or_else(|| "(untitled)".to_string());
|
||||
let original_title = match kind {
|
||||
TmdbKind::Movie => self.original_title,
|
||||
TmdbKind::Tv => self.original_name,
|
||||
};
|
||||
let date = match kind {
|
||||
TmdbKind::Movie => self.release_date,
|
||||
TmdbKind::Tv => self.first_air_date,
|
||||
};
|
||||
let year = date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
TmdbHit {
|
||||
tmdbid: self.id,
|
||||
title,
|
||||
original_title,
|
||||
original_language: self.original_language,
|
||||
overview: self.overview,
|
||||
year,
|
||||
poster_path: self.poster_path,
|
||||
genre_ids: self.genre_ids,
|
||||
// search-list items carry no AKAs (a details/append fetch does)
|
||||
alt_titles: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map an ISO 639-1 (2-letter) code to its ISO 639-3 (`lang3`) equivalent,
|
||||
/// matching the store's convention (`eng`, `jpn`, `fra`; the 639-2/T variant
|
||||
/// where B/T differ, e.g. `deu` not `ger`). Covers the languages TMDB
|
||||
/// commonly returns; unknown codes return `None` so the caller skips the
|
||||
/// `titles/{lang3}` write rather than persisting a non-`lang3` key.
|
||||
pub(crate) fn iso639_1_to_3(code: &str) -> Option<&'static str> {
|
||||
Some(match code.to_ascii_lowercase().as_str() {
|
||||
"en" => "eng",
|
||||
"ja" => "jpn",
|
||||
"fr" => "fra",
|
||||
"de" => "deu",
|
||||
"es" => "spa",
|
||||
"it" => "ita",
|
||||
"ru" => "rus",
|
||||
"ko" => "kor",
|
||||
"zh" => "zho",
|
||||
"pt" => "por",
|
||||
"nl" => "nld",
|
||||
"sv" => "swe",
|
||||
"no" => "nor",
|
||||
"da" => "dan",
|
||||
"fi" => "fin",
|
||||
"pl" => "pol",
|
||||
"tr" => "tur",
|
||||
"ar" => "ara",
|
||||
"hi" => "hin",
|
||||
"th" => "tha",
|
||||
"vi" => "vie",
|
||||
"id" => "ind",
|
||||
"cs" => "ces",
|
||||
"el" => "ell",
|
||||
"he" => "heb",
|
||||
"hu" => "hun",
|
||||
"ro" => "ron",
|
||||
"uk" => "ukr",
|
||||
"fa" => "fas",
|
||||
"ms" => "msa",
|
||||
"tl" => "tgl",
|
||||
"ca" => "cat",
|
||||
"nb" => "nob",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
// `Clone` is required so the single-flight `Shared` future (whose `Output`
|
||||
// must be `Clone`) can hand the same hit to every coalesced caller; serde so
|
||||
// hits persist in the redb TMDB-search cache.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbHit {
|
||||
pub(crate) tmdbid: u64,
|
||||
pub(crate) title: String,
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) year: Option<u16>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)] // surfaced as CSV of ids; name resolution would need a second call
|
||||
pub(crate) genre_ids: Vec<u32>,
|
||||
/// TMDB alternative / AKA titles (romaji + foreign), used by the id-based-job
|
||||
/// relevance guard so a differently-titled release still resembles the show.
|
||||
/// Empty for hits built from a search list (which carries no AKAs).
|
||||
#[serde(default)]
|
||||
pub(crate) alt_titles: Vec<String>,
|
||||
}
|
||||
|
||||
impl TmdbHit {
|
||||
/// TMDB `original_language` (ISO 639-1) mapped to the store's `lang3`
|
||||
/// (ISO 639-3), or `None` when the language is outside the common set or
|
||||
/// absent. Used to file `original_title` under `titles/{lang3}` (§3).
|
||||
pub(crate) fn original_lang3(&self) -> Option<&'static str> {
|
||||
iso639_1_to_3(self.original_language.as_deref()?.trim())
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative TV structure from `GET /3/tv/{id}`, used to bounds-check
|
||||
/// a title-parsed season/episode. Only the two structural fields are
|
||||
/// decoded; everything else in the (large) details payload is ignored.
|
||||
// `Clone` + `Serialize` added so details persist in the redb TMDB-tvdetails
|
||||
// cache and can be cheaply handed around.
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbTvDetails {
|
||||
#[serde(default)]
|
||||
pub(crate) number_of_seasons: u32,
|
||||
/// One entry per season TMDB knows about, including season 0
|
||||
/// ("Specials"). `episode_count` is the released-episode total.
|
||||
#[serde(default)]
|
||||
pub(crate) seasons: Vec<TmdbSeasonSummary>,
|
||||
// -- Display fields ------------------------------------------------------
|
||||
// Decoded from the same `GET /3/tv/{id}` payload so an anchored TV record
|
||||
// (known tmdbid) enriches directly from its canonical entry instead of a
|
||||
// fuzzy title search. Old cache entries (written before these fields
|
||||
// existed) deserialize them to empty/None; [`TmdbTvDetails::has_display`]
|
||||
// detects that so the cached-fetch path can self-heal with one refetch.
|
||||
#[serde(default)]
|
||||
pub(crate) name: String,
|
||||
#[serde(default)]
|
||||
pub(crate) original_name: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) first_air_date: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
/// AKA titles from `?append_to_response=alternative_titles`. `None` on a
|
||||
/// pre-AKA cache entry → [`TmdbTvDetails::has_akas`] is false so the cached
|
||||
/// fetch self-heals with one refetch (same pattern as the display fields).
|
||||
#[serde(default)]
|
||||
pub(crate) alternative_titles: Option<TmdbAltTitles>,
|
||||
}
|
||||
|
||||
impl TmdbTvDetails {
|
||||
/// True when the cached payload carries the display fields (i.e. was
|
||||
/// written by the current decoder). A `false` here on a cache hit means a
|
||||
/// pre-display entry — the caller refetches once to upgrade it.
|
||||
pub(crate) fn has_display(&self) -> bool {
|
||||
!self.name.trim().is_empty()
|
||||
}
|
||||
|
||||
/// True when the AKA field was populated by the current (append-enabled)
|
||||
/// decoder. A `false` on a cache hit means a pre-AKA entry → the cached
|
||||
/// fetch refetches once to upgrade it (mirrors [`has_display`]).
|
||||
pub(crate) fn has_akas(&self) -> bool {
|
||||
self.alternative_titles.is_some()
|
||||
}
|
||||
|
||||
/// The show's canonical + original + AKA titles, for the id-based-job
|
||||
/// relevance guard.
|
||||
pub(crate) fn guard_titles(&self) -> Vec<String> {
|
||||
let mut out = vec![self.name.clone()];
|
||||
if let Some(o) = &self.original_name {
|
||||
out.push(o.clone());
|
||||
}
|
||||
if let Some(a) = &self.alternative_titles {
|
||||
out.extend(a.all());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Build a [`TmdbHit`] (resolved by id) from the canonical TV entry, for
|
||||
/// the anchored enrichment path. `None` when display fields are absent.
|
||||
pub(crate) fn as_hit(&self, tmdbid: u64) -> Option<TmdbHit> {
|
||||
if !self.has_display() {
|
||||
return None;
|
||||
}
|
||||
let year = self
|
||||
.first_air_date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
Some(TmdbHit {
|
||||
tmdbid,
|
||||
title: self.name.clone(),
|
||||
original_title: self.original_name.clone(),
|
||||
original_language: self.original_language.clone(),
|
||||
overview: self.overview.clone(),
|
||||
year,
|
||||
poster_path: self.poster_path.clone(),
|
||||
genre_ids: Vec::new(),
|
||||
alt_titles: self
|
||||
.alternative_titles
|
||||
.as_ref()
|
||||
.map(TmdbAltTitles::all)
|
||||
.unwrap_or_default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Alternative / AKA titles from `?append_to_response=alternative_titles`.
|
||||
/// TMDB nests them under `results` for `/tv/{id}` and under `titles` for
|
||||
/// `/movie/{id}`; both entries carry a `title`. Decoded so the id-based-job
|
||||
/// relevance guard ([`super::torznab::apply_record_tag`]) can match a
|
||||
/// romaji/foreign release title (e.g. `Sousou no Frieren`) against a show whose
|
||||
/// canonical `name` is the English one (`Frieren: Beyond Journey's End`).
|
||||
///
|
||||
/// `Option<TmdbAltTitles>` on the details structs doubles as a fetched-marker:
|
||||
/// `None` means "written by a decoder that didn't append AKAs" (an old cache
|
||||
/// entry) and triggers a one-shot self-heal refetch; `Some` (even empty) means
|
||||
/// "AKAs were fetched, this show simply has none".
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbAltTitles {
|
||||
#[serde(default)]
|
||||
pub(crate) results: Vec<TmdbAltTitle>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) titles: Vec<TmdbAltTitle>, // movie
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbAltTitle {
|
||||
#[serde(default)]
|
||||
pub(crate) title: String,
|
||||
}
|
||||
|
||||
impl TmdbAltTitles {
|
||||
/// The non-empty AKA titles across both the TV (`results`) and movie
|
||||
/// (`titles`) shapes.
|
||||
pub(crate) fn all(&self) -> Vec<String> {
|
||||
self.results
|
||||
.iter()
|
||||
.chain(self.titles.iter())
|
||||
.map(|t| t.title.trim().to_string())
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// TMDB cross-database ids from `GET /3/{tv,movie}/{id}/external_ids`. Only the
|
||||
/// two the anchored torznab path needs are decoded.
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbExternalIds {
|
||||
#[serde(default)]
|
||||
pub(crate) tvdb_id: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub(crate) imdb_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Authoritative movie structure from `GET /3/movie/{id}`. Decodes only the
|
||||
/// display fields the anchored path needs; `imdb_id` (movie details carry it
|
||||
/// inline, no separate `external_ids` call) feeds the `t=movie&imdbid=` query
|
||||
/// fallback.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct TmdbMovieDetails {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) release_date: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) alternative_titles: Option<TmdbAltTitles>,
|
||||
}
|
||||
|
||||
impl TmdbMovieDetails {
|
||||
pub(crate) fn into_hit(self) -> TmdbHit {
|
||||
let year = self
|
||||
.release_date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
let alt_titles = self
|
||||
.alternative_titles
|
||||
.as_ref()
|
||||
.map(TmdbAltTitles::all)
|
||||
.unwrap_or_default();
|
||||
TmdbHit {
|
||||
tmdbid: self.id,
|
||||
title: self.title.unwrap_or_else(|| "(untitled)".to_string()),
|
||||
original_title: self.original_title,
|
||||
original_language: self.original_language,
|
||||
overview: self.overview,
|
||||
year,
|
||||
poster_path: self.poster_path,
|
||||
genre_ids: Vec::new(),
|
||||
alt_titles,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbMultiResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) results: Vec<TmdbMultiItem>,
|
||||
}
|
||||
|
||||
/// One `search/multi` result. `media_type` discriminates movie / tv / person;
|
||||
/// we anchor only on movie/tv.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct TmdbMultiItem {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) media_type: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) popularity: f64,
|
||||
}
|
||||
|
||||
impl TmdbMultiItem {
|
||||
pub(crate) fn kind(&self) -> Option<TmdbKind> {
|
||||
match self.media_type.as_deref() {
|
||||
Some("movie") => Some(TmdbKind::Movie),
|
||||
Some("tv") => Some(TmdbKind::Tv),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
pub(crate) fn display_title(&self) -> &str {
|
||||
self.title.as_deref().or(self.name.as_deref()).unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowercase alphanumeric word tokens of `s` (separators dropped). Shared by
|
||||
/// the principal-search confidence check.
|
||||
pub(crate) fn norm_tokens(s: &str) -> Vec<String> {
|
||||
s.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(|t| t.to_lowercase())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Pick up to `n` **confident** anchors from `search/multi` hits for `query`,
|
||||
/// most-popular first (or `[]` → the caller falls back to today's generic
|
||||
/// `t=search&q=`). For each movie/tv hit, accept it only if its title shares at
|
||||
/// least half of the query's word tokens (the per-candidate relevance gate),
|
||||
/// then rank the survivors by popularity and take the top `n`. This binds a
|
||||
/// vague keyword like "black" to the *several* real shows TMDB returns (Black
|
||||
/// Mirror, Black Butler, Black Lagoon, …) instead of hijacking the whole query
|
||||
/// onto the single most-popular title, while still dropping unrelated noise
|
||||
/// (e.g. "zzz qqq www" → nothing). `n == 1` recovers the old single-anchor
|
||||
/// "principal_confident" behaviour (modulo the gate being applied before the
|
||||
/// popularity pick rather than after).
|
||||
pub(crate) fn principal_top_n<'a>(
|
||||
hits: &'a [TmdbMultiItem],
|
||||
query: &str,
|
||||
n: usize,
|
||||
) -> Vec<&'a TmdbMultiItem> {
|
||||
let q_tokens = norm_tokens(query);
|
||||
if q_tokens.is_empty() || n == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut cands: Vec<&TmdbMultiItem> = hits
|
||||
.iter()
|
||||
.filter(|h| h.kind().is_some())
|
||||
.filter(|h| {
|
||||
let t_tokens = norm_tokens(h.display_title());
|
||||
let overlap = q_tokens.iter().filter(|t| t_tokens.contains(t)).count();
|
||||
(overlap as f64) >= (q_tokens.len() as f64) * 0.5
|
||||
})
|
||||
.collect();
|
||||
cands.sort_by(|a, b| {
|
||||
b.popularity
|
||||
.partial_cmp(&a.popularity)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
cands.truncate(n);
|
||||
cands
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbSeasonSummary {
|
||||
#[serde(default)]
|
||||
pub(crate) season_number: i64,
|
||||
#[serde(default)]
|
||||
pub(crate) episode_count: u32,
|
||||
}
|
||||
|
||||
/// Verdict of [`season_episode_bounds`] — how a title-parsed `(season,
|
||||
/// episode)` lines up against TMDB's actual structure.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum SeasonEpisodeBounds {
|
||||
/// Consistent with TMDB, or unverifiable — keep the record as-is.
|
||||
Ok,
|
||||
/// The parsed season sits past TMDB's known seasons and isn't otherwise
|
||||
/// listed in `seasons[]`. The canonical case is anime where fansubbers
|
||||
/// number a later cour `S2` of a show TMDB models as a **single** season
|
||||
/// (Frieren, Jujutsu Kaisen, …). The TMDB match itself is sound — only the
|
||||
/// season number is a naming-convention artifact — so the caller keeps the
|
||||
/// record but strips the misleading `season` (the frontend buckets
|
||||
/// season-less episodes into a flat "Episodes" list).
|
||||
SeasonOverflow,
|
||||
/// The season IS a real TMDB season, but the parsed episode runs past that
|
||||
/// season's `episode_count`. The canonical case is absolute-numbered anime
|
||||
/// that an indexer has relabelled onto season 1 (`S01E96` on a 28-episode
|
||||
/// show): the `S01` is a numbering artifact, not a trustworthy token, so —
|
||||
/// exactly like [`SeasonOverflow`] — the caller strips the season and keeps
|
||||
/// the record (season-less episode, bucketed under the show's tmdbid). We
|
||||
/// deliberately do NOT drop it: an out-of-bounds episode means the season
|
||||
/// was never reliable, not that the whole release is bogus.
|
||||
EpisodeOverflow,
|
||||
/// A contradiction neither overflow allowance can explain — currently only a
|
||||
/// negative season (codec noise like `x.265` → "S0E265" is caught upstream;
|
||||
/// a genuinely garbled parse). The title parse is wrong — the caller drops
|
||||
/// the record.
|
||||
Contradiction,
|
||||
}
|
||||
|
||||
/// Pure bounds check: is a title-parsed `(season, episode)` consistent
|
||||
/// with what TMDB says the show actually has? Deliberately lenient —
|
||||
/// it only returns a verdict other than [`SeasonEpisodeBounds::Ok`] on a
|
||||
/// *positive* divergence, so anything ambiguous or unverifiable is accepted:
|
||||
///
|
||||
/// - No parsed season → [`Ok`]. Episode-only titles (absolute-numbered
|
||||
/// anime, trailing `- 117`) are legitimately unbounded; the season-
|
||||
/// based bound doesn't apply.
|
||||
/// - TMDB gave us no structure (`number_of_seasons == 0` and no
|
||||
/// `seasons[]`) → [`Ok`]; we can't contradict what we don't know.
|
||||
/// - Season exceeds `number_of_seasons` AND isn't otherwise listed in
|
||||
/// `seasons[]` → [`SeasonOverflow`] (the `S2`-of-a-1-TMDB-season anime
|
||||
/// case; keep but strip `season`).
|
||||
/// - Negative season → [`Contradiction`].
|
||||
/// - Episode exceeds the matched season's `episode_count` (when that
|
||||
/// count is known) → [`EpisodeOverflow`]. This fires for explicit `SxxEyy`
|
||||
/// titles whose episode overruns the season — overwhelmingly absolute-
|
||||
/// numbered anime relabelled onto season 1 — so we strip the season and keep
|
||||
/// the record rather than dropping it.
|
||||
///
|
||||
/// [`Ok`]: SeasonEpisodeBounds::Ok
|
||||
/// [`SeasonOverflow`]: SeasonEpisodeBounds::SeasonOverflow
|
||||
/// [`EpisodeOverflow`]: SeasonEpisodeBounds::EpisodeOverflow
|
||||
/// [`Contradiction`]: SeasonEpisodeBounds::Contradiction
|
||||
pub(crate) fn season_episode_bounds(
|
||||
details: &TmdbTvDetails,
|
||||
season: Option<i64>,
|
||||
episode: Option<i64>,
|
||||
) -> SeasonEpisodeBounds {
|
||||
use SeasonEpisodeBounds::*;
|
||||
let season = match season {
|
||||
Some(s) => s,
|
||||
None => return Ok,
|
||||
};
|
||||
if details.number_of_seasons == 0 && details.seasons.is_empty() {
|
||||
return Ok;
|
||||
}
|
||||
if season < 0 {
|
||||
return Contradiction;
|
||||
}
|
||||
let matched = details.seasons.iter().find(|s| s.season_number == season);
|
||||
if season > details.number_of_seasons as i64 && matched.is_none() {
|
||||
return SeasonOverflow;
|
||||
}
|
||||
if let (Some(ep), Some(sm)) = (episode, matched) {
|
||||
if sm.episode_count > 0 && ep > sm.episode_count as i64 {
|
||||
return EpisodeOverflow;
|
||||
}
|
||||
}
|
||||
Ok
|
||||
}
|
||||
|
||||
/// True when `episode` is a valid episode of `season` per TMDB's per-season
|
||||
/// `episode_count` (the season must be listed and the count known). Used to
|
||||
/// decide whether a bare title number is an in-season episode before falling
|
||||
/// back to absolute-number interpretation.
|
||||
pub(crate) fn episode_in_season(details: &TmdbTvDetails, season: i64, episode: i64) -> bool {
|
||||
details
|
||||
.seasons
|
||||
.iter()
|
||||
.find(|s| s.season_number == season)
|
||||
.map(|s| s.episode_count > 0 && episode >= 1 && episode <= s.episode_count as i64)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
use crate::consts::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod season_bounds_tests {
|
||||
use super::*;
|
||||
|
||||
fn details(num_seasons: u32, seasons: &[(i64, u32)]) -> TmdbTvDetails {
|
||||
TmdbTvDetails {
|
||||
number_of_seasons: num_seasons,
|
||||
seasons: seasons
|
||||
.iter()
|
||||
.map(|&(season_number, episode_count)| TmdbSeasonSummary {
|
||||
season_number,
|
||||
episode_count,
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_ok_within_season() {
|
||||
let d = details(2, &[(1, 26), (2, 24)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(1), Some(5)), SeasonEpisodeBounds::Ok);
|
||||
assert_eq!(season_episode_bounds(&d, Some(2), Some(24)), SeasonEpisodeBounds::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_no_season_or_unknown_structure_is_ok() {
|
||||
let d = details(2, &[(1, 26), (2, 24)]);
|
||||
assert_eq!(season_episode_bounds(&d, None, Some(999)), SeasonEpisodeBounds::Ok);
|
||||
let unknown = details(0, &[]);
|
||||
assert_eq!(season_episode_bounds(&unknown, Some(5), Some(40)), SeasonEpisodeBounds::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_season_past_total_is_overflow() {
|
||||
// Fansub "S2" of a show TMDB models as a single season → strip-and-keep.
|
||||
let d = details(1, &[(1, 28)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(2), Some(3)), SeasonEpisodeBounds::SeasonOverflow);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_episode_past_matched_season_is_episode_overflow() {
|
||||
// "S1E27" on a show whose season 1 has 26 episodes → the `S01` is an
|
||||
// absolute-numbering artifact, not a trustworthy token. The enrich path
|
||||
// strips the season and KEEPS the record (season-less absolute episode),
|
||||
// rather than dropping it.
|
||||
let d = details(4, &[(1, 26), (2, 13), (3, 13), (4, 12)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(1), Some(27)), SeasonEpisodeBounds::EpisodeOverflow);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_negative_season_is_contradiction() {
|
||||
// A genuinely garbled parse (negative season) is still a hard drop.
|
||||
let d = details(4, &[(1, 26), (2, 13), (3, 13), (4, 12)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(-1), None), SeasonEpisodeBounds::Contradiction);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
//! End-to-end contract test for the card feeder's HTTP surface: boots the real
|
||||
//! `serve_feeders` router (card-feeder hosting the TMDB plugin) on an ephemeral
|
||||
//! port, pointed at a wiremock TMDB, and drives `/manifest`, `/query` and
|
||||
//! `/compute` over HTTP. Deterministic — no live TMDB.
|
||||
//!
|
||||
//! What this pins is the *card contract* specifically:
|
||||
//! - the manifest advertises `fileType=card`, which is the only reason
|
||||
//! meta-search routes a `fileType:card` query here;
|
||||
//! - a query yields records carrying both type axes plus the whole id bag;
|
||||
//! - `/compute` yields a **byte-less** `card_locator` outcome — the shape that
|
||||
//! lands in the gateway core's metadata-only auto-store branch.
|
||||
|
||||
use card_feeder::tmdb::TmdbCardPlugin;
|
||||
use meta_feeder_sdk::{
|
||||
configure_plugins, router, ComputeRequest, ComputeResponse, GatewayQuery, HashKindDto,
|
||||
ManifestResponse, QueryRequest, QueryResponse,
|
||||
};
|
||||
use wiremock::matchers::{method, path, query_param};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
/// Spawn the feeder router (TMDB pointed at `upstream`) on an ephemeral port.
|
||||
/// The TempDir is returned so the per-plugin cache outlives the test.
|
||||
async fn spawn_feeder(upstream: &MockServer) -> (String, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let plugin = TmdbCardPlugin::with_api_base("test-token", upstream.uri(), upstream.uri());
|
||||
let plugins = configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure plugins");
|
||||
let app = router(plugins, "card-feeder-test".to_string(), dir.path());
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind ephemeral");
|
||||
let addr = listener.local_addr().expect("local_addr");
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.expect("serve");
|
||||
});
|
||||
(format!("http://{addr}"), dir)
|
||||
}
|
||||
|
||||
/// `search/multi` → one confident TV hit for "frieren".
|
||||
fn multi_json() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"results": [{
|
||||
"id": 95479,
|
||||
"media_type": "tv",
|
||||
"name": "Frieren: Beyond Journey's End",
|
||||
"original_name": "\u{845}\u{9001}\u{306e}\u{30d5}\u{30ea}\u{30fc}\u{30ec}\u{30f3}",
|
||||
"overview": "The story follows the elf mage Frieren.",
|
||||
"poster_path": "/poster.jpg",
|
||||
"first_air_date": "2023-09-29",
|
||||
"popularity": 300.0,
|
||||
"vote_count": 900
|
||||
}]
|
||||
})
|
||||
}
|
||||
|
||||
fn tv_details_json() -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"id": 95479,
|
||||
"name": "Frieren: Beyond Journey's End",
|
||||
"original_name": "\u{845}\u{9001}\u{306e}\u{30d5}\u{30ea}\u{30fc}\u{30ec}\u{30f3}",
|
||||
"original_language": "ja",
|
||||
"overview": "The story follows the elf mage Frieren.",
|
||||
"first_air_date": "2023-09-29",
|
||||
"poster_path": "/poster.jpg",
|
||||
"number_of_seasons": 1,
|
||||
"seasons": [{"season_number": 1, "episode_count": 28}],
|
||||
"alternative_titles": { "results": [{"title": "Sousou no Frieren"}] }
|
||||
})
|
||||
}
|
||||
|
||||
fn external_ids_json() -> serde_json::Value {
|
||||
serde_json::json!({ "imdb_id": "tt22248376", "tvdb_id": 367189 })
|
||||
}
|
||||
|
||||
async fn mount_tmdb(upstream: &MockServer) {
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/search/multi"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(multi_json()))
|
||||
.mount(upstream)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/tv/95479"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(tv_details_json()))
|
||||
.mount(upstream)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/tv/95479/external_ids"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(external_ids_json()))
|
||||
.mount(upstream)
|
||||
.await;
|
||||
}
|
||||
|
||||
fn card_query(text: &str) -> GatewayQuery {
|
||||
let mut q = GatewayQuery::from_free_text(text);
|
||||
q.filters
|
||||
.insert("fileType".to_string(), vec!["card".to_string()]);
|
||||
q
|
||||
}
|
||||
|
||||
/// The routing declaration. `served_file_types = ["card"]` is what makes
|
||||
/// meta-search fan a `fileType:card` query out to this gateway at all.
|
||||
#[tokio::test]
|
||||
async fn manifest_advertises_the_card_file_type() {
|
||||
let upstream = MockServer::start().await;
|
||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||
|
||||
let manifest: ManifestResponse = reqwest::Client::new()
|
||||
.get(format!("{base}/manifest"))
|
||||
.send()
|
||||
.await
|
||||
.expect("GET /manifest")
|
||||
.json()
|
||||
.await
|
||||
.expect("decode manifest");
|
||||
|
||||
assert_eq!(manifest.plugins.len(), 1);
|
||||
let p = &manifest.plugins[0];
|
||||
assert_eq!(p.id, "tmdb");
|
||||
assert_eq!(p.served_file_types, vec!["card".to_string()]);
|
||||
assert_eq!(
|
||||
p.served_content_kinds,
|
||||
vec!["movie".to_string(), "series".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_returns_a_card_with_both_axes_and_the_id_bag() {
|
||||
let upstream = MockServer::start().await;
|
||||
mount_tmdb(&upstream).await;
|
||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||
|
||||
let resp: QueryResponse = reqwest::Client::new()
|
||||
.post(format!("{base}/query"))
|
||||
.json(&QueryRequest {
|
||||
upstream_id: "tmdb".to_string(),
|
||||
query: card_query("frieren"),
|
||||
max_results: 10,
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect("POST /query")
|
||||
.json()
|
||||
.await
|
||||
.expect("decode query response");
|
||||
|
||||
assert_eq!(resp.records.len(), 1, "one confident TMDB hit → one card");
|
||||
let f = &resp.records[0].fields;
|
||||
assert_eq!(f["fileType"], "card");
|
||||
assert_eq!(f["contentKind"], "series");
|
||||
assert_eq!(f["title"], "Frieren: Beyond Journey's End");
|
||||
// The id bag — phase 2 selects from this rather than assuming `tmdbid`.
|
||||
assert_eq!(f["tmdbid"], "95479");
|
||||
assert_eq!(f["tvdbid"], "367189");
|
||||
assert_eq!(f["imdbid"], "tt22248376");
|
||||
// Display trio that meta-watch's quality gate requires.
|
||||
assert!(f.contains_key("description/eng"));
|
||||
assert!(f["poster_url"].contains("poster.jpg"));
|
||||
// The record id is the CID preimage.
|
||||
assert_eq!(resp.records[0].record_id, "tmdb:tv:95479");
|
||||
}
|
||||
|
||||
/// The core of the design: a card resolves to a locator with **no bytes**.
|
||||
/// That `bytes_b64: None` is what routes it to `store_metadata_only` in the
|
||||
/// gateway core's three-branch auto-store.
|
||||
#[tokio::test]
|
||||
async fn compute_returns_a_byteless_card_locator() {
|
||||
let upstream = MockServer::start().await;
|
||||
mount_tmdb(&upstream).await;
|
||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||
|
||||
let resp: ComputeResponse = reqwest::Client::new()
|
||||
.post(format!("{base}/compute"))
|
||||
.json(&ComputeRequest {
|
||||
upstream_id: "tmdb".to_string(),
|
||||
record_id: "tmdb:tv:95479".to_string(),
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect("POST /compute")
|
||||
.json()
|
||||
.await
|
||||
.expect("decode compute response");
|
||||
|
||||
assert_eq!(resp.outcomes.len(), 1);
|
||||
let o = &resp.outcomes[0];
|
||||
assert!(matches!(o.hash_kind, HashKindDto::CardLocator));
|
||||
assert!(o.bytes_b64.is_none(), "a card has no bytes, ever");
|
||||
assert!(o.file_extension.is_none());
|
||||
assert!(o.record.is_some(), "the record IS the payload");
|
||||
// Deterministic in (source, id) — any peer derives the same address.
|
||||
assert_eq!(
|
||||
o.hash,
|
||||
meta_feeder_sdk::hash::compute_card_cid("tmdb", "tv:95479").unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
/// A card query never reaches an indexer, and a *typed* query for something the
|
||||
/// card tier does not serve short-circuits before TMDB is touched at all.
|
||||
#[tokio::test]
|
||||
async fn a_video_query_does_not_hit_tmdb() {
|
||||
let upstream = MockServer::start().await;
|
||||
// Deliberately mount nothing: any outbound call would 404 and surface as an
|
||||
// empty result, but the assertion below is that we never even get there.
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/search/multi"))
|
||||
.and(query_param("query", "frieren"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(0)
|
||||
.mount(&upstream)
|
||||
.await;
|
||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||
|
||||
let mut q = GatewayQuery::from_free_text("frieren");
|
||||
q.filters
|
||||
.insert("fileType".to_string(), vec!["video".to_string()]);
|
||||
|
||||
let resp: QueryResponse = reqwest::Client::new()
|
||||
.post(format!("{base}/query"))
|
||||
.json(&QueryRequest {
|
||||
upstream_id: "tmdb".to_string(),
|
||||
query: q,
|
||||
max_results: 10,
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect("POST /query")
|
||||
.json()
|
||||
.await
|
||||
.expect("decode query response");
|
||||
|
||||
assert!(resp.records.is_empty());
|
||||
// `expect(0)` above is asserted on MockServer drop.
|
||||
}
|
||||
Reference in New Issue
Block a user