inital commit
This commit is contained in:
Worph
@@ -0,0 +1,48 @@
|
||||
[package]
|
||||
name = "tribler-feeder"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
|
||||
# Tribler feeder sidecar: bridges the decentralized Tribler (IPv8 overlay)
|
||||
# torrent network into the MetaMesh network. Split out of `meta-feeder-torrent`
|
||||
# so it is a single-plugin feeder (its dashboard config panel resolves cleanly,
|
||||
# unlike a multi-plugin feeder whose one /config surface can only expose one
|
||||
# plugin). Carries a copy of the shared "torrent-core" TMDB catalog-discovery
|
||||
# stack (the part that CANNOT be a container plugin — home-row browse has no
|
||||
# per-CID plugin equivalent); the librqbit BitTorrent fetch stays in
|
||||
# meta-feeder-torrent (tribler enumerates files via the Tribler core's metainfo,
|
||||
# not librqbit). Like every feeder it is meta-core-free + blockstore-free.
|
||||
|
||||
[lib]
|
||||
name = "tribler_feeder"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "tribler-feeder"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" }
|
||||
|
||||
tokio = { version = "1.40", features = ["full"] }
|
||||
axum = { version = "0.7", features = ["json"] }
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
thiserror = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
|
||||
bytes = "1"
|
||||
futures = "0.3"
|
||||
regex = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
|
||||
|
||||
[dev-dependencies]
|
||||
wiremock = "0.6"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.40", features = ["full", "test-util"] }
|
||||
@@ -0,0 +1,55 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# tribler-feeder — Tribler (IPv8 overlay) feeder sidecar. Multi-stage build.
|
||||
#
|
||||
# Build context must be this repo root (packages/plugins/meta-feeder-tribler) so
|
||||
# the `meta-feeder-sdk` path dependency resolves. Same BuildKit cache-mount
|
||||
# strategy as the other feeders. No librqbit here (tribler enumerates files via
|
||||
# the Tribler core's metainfo), so the cold build is lighter than the torrent
|
||||
# feeder's.
|
||||
# =============================================================================
|
||||
|
||||
FROM rust:1.89-slim-bookworm AS builder
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
ca-certificates \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||
COPY crates/ crates/
|
||||
COPY feeder-plugin/ feeder-plugin/
|
||||
|
||||
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=meta-feeder-tribler-target,target=/build/target \
|
||||
cargo build --release --bin tribler-feeder \
|
||||
&& cp target/release/tribler-feeder /tribler-feeder
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /tribler-feeder /usr/local/bin/tribler-feeder
|
||||
|
||||
RUN useradd --system --uid 10001 --no-create-home metafeeder \
|
||||
&& mkdir -p /data/meta-feeder \
|
||||
&& chown metafeeder:metafeeder /data/meta-feeder
|
||||
|
||||
USER metafeeder
|
||||
WORKDIR /data/meta-feeder
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
|
||||
META_FEEDER_STATE_DIR=/data/meta-feeder \
|
||||
RUST_LOG=info
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/tribler-feeder"]
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Shared constants for the torznab plugin module.
|
||||
//!
|
||||
//! These symbols are referenced across `mod.rs`, `tmdb.rs` and `enrich.rs`
|
||||
//! but were never committed when the plugin was split from a single file into
|
||||
//! this directory module — the crate failed to compile (`E0425: cannot find
|
||||
//! value ...`). Values below are sensible defaults; review them against the
|
||||
//! intended torznab/TMDB contract (endpoints + env-var names especially).
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// User-Agent for outbound HTTP from the torznab plugin (TMDB + indexers).
|
||||
/// Mirrors the per-plugin convention in `plugins/{arxiv,giphy,gutenberg}.rs`.
|
||||
pub(crate) const USER_AGENT: &str = concat!(
|
||||
"meta-share/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" (gateway:torznab)"
|
||||
);
|
||||
|
||||
/// TMDB REST API base (v3).
|
||||
pub(crate) const TMDB_API_BASE: &str = "https://api.themoviedb.org/3";
|
||||
/// TMDB image CDN base; callers append a size segment (e.g. `/w500/<path>`).
|
||||
pub(crate) const TMDB_IMAGE_BASE: &str = "https://image.tmdb.org/t/p";
|
||||
/// TMDB poster size segment inserted between [`TMDB_IMAGE_BASE`] and the
|
||||
/// poster path. Mandatory — TMDB 404s a path with no size. `w500` is the
|
||||
/// poster-grid sweet spot (≈122 KB vs `original`'s multi-MB).
|
||||
pub(crate) const TMDB_POSTER_SIZE: &str = "w500";
|
||||
|
||||
/// Generic outbound HTTP timeout (seconds). Raised 100x (30 -> 3000) for
|
||||
/// completeness — the gateway runs inside a long, streaming search, not a
|
||||
/// sub-5s one; an outbound call that's still making progress shouldn't be cut.
|
||||
pub(crate) const HTTP_TIMEOUT_SECS: u64 = 3000;
|
||||
/// TMDB search request timeout (seconds). Raised 100x (10 -> 1000) — TMDB
|
||||
/// cross-ref enrichment is part of result completeness and needs time.
|
||||
pub(crate) const TMDB_SEARCH_TIMEOUT_SECS: u64 = 1000;
|
||||
/// TMDB poster download timeout (seconds). Raised 100x (30 -> 3000).
|
||||
pub(crate) const TMDB_POSTER_TIMEOUT_SECS: u64 = 3000;
|
||||
/// Default Retry-After (seconds) applied when TMDB returns HTTP 429 with no
|
||||
/// (or an unparseable) `Retry-After` header.
|
||||
pub(crate) const TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS: u64 = 5;
|
||||
/// Maximum time to wait for a TMDB enrichment permit before giving up (seconds).
|
||||
/// Raised 100x (30 -> 3000) — under a deep search many records contend for the
|
||||
/// 4 enrichment permits; giving up early drops enrichment from the tail.
|
||||
pub(crate) const TMDB_ENRICH_WAIT_DEADLINE_SECS: u64 = 3000;
|
||||
/// Maximum number of concurrent TMDB enrichment requests.
|
||||
pub(crate) const TMDB_ENRICH_CONCURRENCY: usize = 4;
|
||||
/// Per-indexer torznab query timeout (seconds). This is the cap that actually
|
||||
/// guillotines slow trackers on the primary *streaming* gateway path (the 5s
|
||||
/// `GATEWAY_CALL_TIMEOUT` only bounds the one-shot fallback). Raised 100x
|
||||
/// (3 -> 300) for completeness so a slow-but-alive indexer is waited for
|
||||
/// rather than silently dropped from the merge. Stays <= the (also-raised)
|
||||
/// 500s one-shot `GATEWAY_CALL_TIMEOUT`. NOTE: `search_probe_timeout()` reuses
|
||||
/// this value, so reachability probes against a *dead* indexer now hang up to
|
||||
/// 300s — acceptable for a completeness-first posture, but the dual use is why
|
||||
/// this isn't split into its own probe constant yet.
|
||||
pub(crate) const PER_INDEXER_QUERY_TIMEOUT_SECS: u64 = 300;
|
||||
|
||||
/// Max concurrent `.torrent` downloads while resolving one indexer's results.
|
||||
/// Bounds the socket burst when a whole result page is torrent-only. Kept low
|
||||
/// (4) because the resolve fetches hit public trackers/torrent caches that
|
||||
/// reset connections under a wide burst — the same load that makes Prowlarr's
|
||||
/// own indexer calls flake. Indexer *search* GETs are now serialized one at a
|
||||
/// time through the shared [`INDEXER_LIMITER_RPS`] token bucket, so the peak
|
||||
/// socket count is just this many (the in-flight page's resolves), not a
|
||||
/// product across concurrent indexers.
|
||||
pub(crate) const TORRENT_RESOLVE_CONCURRENCY: usize = 4;
|
||||
|
||||
/// Defensive cap on how many torrent-only items we attempt to resolve per
|
||||
/// indexer response. The rest are dropped (logged) — at search latency,
|
||||
/// resolving an unbounded page of `.torrent` files would blow the per-indexer
|
||||
/// budget. A page rarely needs more than this many resolved results anyway.
|
||||
pub(crate) const TORRENT_RESOLVE_MAX_PER_INDEXER: usize = 30;
|
||||
|
||||
/// Wall-clock budget for the whole torrent-resolution phase of a single
|
||||
/// indexer call. Items not resolved within it are dropped; items resolved so
|
||||
/// far are kept (partial). The gateway returns as soon as resolution
|
||||
/// *completes* — the budget is only the cutoff for a page that can't finish.
|
||||
///
|
||||
/// **Sized for a long, streaming search, not a sub-5s one.** meta-share's
|
||||
/// primary gateway path is a streaming substream (`stream_one_gateway_peer`)
|
||||
/// with **no per-call deadline** — it forwards each hit to the client SSE as
|
||||
/// it arrives until the gateway sends `Done`. (The 5 s `GATEWAY_CALL_TIMEOUT`
|
||||
/// is only the one-shot fallback for gateways that don't speak the streaming
|
||||
/// protocol.) So this can be generous: the binding constraint is meta-share's
|
||||
/// `SEARCH_TIMEOUT_SECS` silence window (set to match in the dev compose), not
|
||||
/// a hard call timeout. 5 min is a safety ceiling on a pathological page;
|
||||
/// a normal XXX page (≤ `TORRENT_RESOLVE_MAX_PER_INDEXER` items at ~1.9 s
|
||||
/// each, 8-wide) finishes in well under 10 s and the gateway returns then.
|
||||
pub(crate) const TORRENT_RESOLVE_BUDGET: Duration = Duration::from_secs(300);
|
||||
|
||||
/// Ceiling on a downloaded `.torrent` body. Real torrent files are KBs; this
|
||||
/// guards against a `<link>` that points at an HTML details page or a
|
||||
/// mislabeled giant download — we skip (don't buffer) anything larger.
|
||||
pub(crate) const TORRENT_FILE_MAX_BYTES: u64 = 5 * 1024 * 1024;
|
||||
|
||||
/// Magnet BEP-9 `list_only` file-list fallback knobs (packs only) — see
|
||||
/// [`crate::bt::TorznabBt::prefetch_torrent_filelists`].
|
||||
///
|
||||
/// HTTP `.torrent` download is always tried first (faster, swarm-free). But
|
||||
/// magnet-only indexers (e.g. Nyaa.si) expose no `.torrent` URL, so a season
|
||||
/// **pack** from them never gets a file list and its per-episode records ship
|
||||
/// without a streamable `cid_btih_v1_file` (see `expand_record_streaming`).
|
||||
/// For packs only — a single file derives its CID from the infohash alone and
|
||||
/// needs no file list — fall back to a bounded BEP-9 `list_only` swarm probe
|
||||
/// off the magnet. Swarm peer/DHT discovery is far slower (and often fruitless
|
||||
/// for dead torrents) than an HTTP fetch, so these caps are deliberately tight:
|
||||
/// the cached result makes the *next* search instant, so a partial pass is fine.
|
||||
pub(crate) const TORRENT_LISTONLY_MAGNET_MAX: usize = 6;
|
||||
/// Concurrent magnet enumerations. Lower than [`TORRENT_RESOLVE_CONCURRENCY`]
|
||||
/// because each holds a live librqbit torrent + peer connections, not a single
|
||||
/// short HTTP GET.
|
||||
pub(crate) const TORRENT_LISTONLY_MAGNET_CONCURRENCY: usize = 2;
|
||||
/// Per-magnet wall-clock cap on the BEP-9 enumeration (tighter than
|
||||
/// `list_files`' own internal full-fetch budget, which is sized for downloads).
|
||||
pub(crate) const TORRENT_LISTONLY_MAGNET_PER_ITEM: Duration = Duration::from_secs(25);
|
||||
/// Overall deadline for the whole magnet-fallback pass; whatever resolved by
|
||||
/// then is kept, the rest fall through to the title-only (un-CID'd) expansion.
|
||||
pub(crate) const TORRENT_LISTONLY_MAGNET_BUDGET: Duration = Duration::from_secs(60);
|
||||
|
||||
/// **Test-only.** When truthy (`1`/`true`/`yes`/`on`), a torznab record that
|
||||
/// fails TMDB enrichment is stubbed with just enough metadata (a `poster`, a
|
||||
/// `description`, and video kind if missing) to pass meta-watch's quality
|
||||
/// gate — simulating "content pollution" so an operator can validate the
|
||||
/// default-on adult-content filter against records that carry `categories/XXX`
|
||||
/// but never match TMDB. **Off by default; never enable in production** — it
|
||||
/// fabricates metadata. See [`super::enrich::stub_unmatched_patch`].
|
||||
pub(crate) const ENV_STUB_UNMATCHED: &str = "META_GATEWAY_PROWLARR_STUB_UNMATCHED";
|
||||
|
||||
/// Timeout used when probing an indexer's search path for reachability.
|
||||
pub(crate) fn search_probe_timeout() -> Duration {
|
||||
Duration::from_secs(PER_INDEXER_QUERY_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
/// Default number of TMDB anchors resolved per query (top-N by popularity).
|
||||
/// For a vague keyword ("black") TMDB returns many real shows; we anchor on the
|
||||
/// top-N rather than collapsing onto the single most popular (the old
|
||||
/// `principal_confident` behaviour that hijacked the query). Each anchor becomes
|
||||
/// its own paged search per indexer, so this multiplies the indexer request
|
||||
/// count — the serialized [`INDEXER_LIMITER_RPS`] bucket keeps that gentle.
|
||||
/// Overridable via `META_GATEWAY_PROWLARR_ANCHOR_TOP_N`.
|
||||
pub(crate) const DEFAULT_ANCHOR_TOP_N: usize = 10;
|
||||
|
||||
/// How many ranked principal-search anchors to persist per query in the redb
|
||||
/// cache. Sized comfortably above any reasonable top-N so flipping the knob
|
||||
/// never needs a cache wipe — reads slice the cached list to the configured N.
|
||||
pub(crate) const CACHED_PRINCIPAL_DEPTH: usize = 30;
|
||||
|
||||
/// Indexer search-request rate limiter (a [`crate::tmdb_budget::TmdbBudget`]
|
||||
/// token bucket shared across the whole torznab fan-out). Public trackers and
|
||||
/// Prowlarr reset/429 under bursts, so search GETs are serialized one at a time
|
||||
/// at ~`INDEXER_LIMITER_RPS` grants/sec with a small `INDEXER_LIMITER_BURST`.
|
||||
/// Adaptive: a 429 (or transient reset/5xx) pauses the bucket via `note_429`.
|
||||
pub(crate) const INDEXER_LIMITER_RPS: f64 = 1.0;
|
||||
/// Burst capacity of the indexer rate limiter (see [`INDEXER_LIMITER_RPS`]).
|
||||
pub(crate) const INDEXER_LIMITER_BURST: f64 = 3.0;
|
||||
/// Backoff applied to the indexer limiter on a transient (connection-reset /
|
||||
/// 5xx) indexer failure that carries no explicit `Retry-After` window.
|
||||
pub(crate) const INDEXER_LIMITER_TRANSIENT_BACKOFF: Duration = Duration::from_secs(2);
|
||||
/// Deadline for acquiring an indexer-limiter token before a page request.
|
||||
/// Generous (completeness-first; meta-share waits its silence window) so a
|
||||
/// token effectively always arrives — `DeadlineExceeded` just drops that page.
|
||||
pub(crate) const INDEXER_LIMITER_WAIT_DEADLINE_SECS: u64 = 3000;
|
||||
|
||||
/// Minimum fraction of a (cleaned) release title's tokens that must be covered
|
||||
/// by an anchor's title before a text-anchored job stamps that anchor's tmdbid
|
||||
/// onto the release. Guards against text false-positives — a release that
|
||||
/// merely *mentions* the anchor title (e.g. "Turning Mecard … Black Mirror",
|
||||
/// 2/≈9 ≈ 0.22) stays unanchored and enriches via its own fuzzy lookup, while a
|
||||
/// real "Black Mirror S05E01 …" (cleans to "Black Mirror", 2/2 = 1.0) is
|
||||
/// stamped. Only applies to fuzzy text jobs; id-based (tvdbid/tmdbid) jobs are
|
||||
/// trusted and bypass the guard. See [`super::anchor::apply_record_tag`].
|
||||
pub(crate) const ANCHOR_TITLE_MIN_COVERAGE: f64 = 0.6;
|
||||
|
||||
/// Backoff before the single retry [`super::TorznabPlugin::send_get_with_retry`]
|
||||
/// makes after a transient (connection-reset / 5xx) indexer failure. Short —
|
||||
/// just enough to let a momentarily-overloaded indexer or a stale pooled
|
||||
/// connection clear before the second attempt.
|
||||
pub(crate) const HTTP_RETRY_BACKOFF: Duration = Duration::from_millis(400);
|
||||
|
||||
/// Defensive ceiling on the per-season fan-out loop. A show with more seasons
|
||||
/// than this (none real; guards a corrupt `number_of_seasons`) only fans out
|
||||
/// the first `MAX_ANCHORED_SEASONS`.
|
||||
pub(crate) const MAX_ANCHORED_SEASONS: u32 = 50;
|
||||
|
||||
/// Per-indexer fetch page size for the depth strategy. Clamped to each
|
||||
/// indexer's declared Prowlarr `capabilities.limitsMax`. Kept small/light so a
|
||||
/// single Prowlarr `indexerIds=<one>` call behaves like the Prowlarr UI (one
|
||||
/// tracker, a modest page) instead of one heavy `indexerIds=-2&limit=2500`
|
||||
/// aggregate — that oversized limit was both futile (every public tracker caps
|
||||
/// at ~100) and the source of the gateway→Prowlarr connection resets under load.
|
||||
pub(crate) const TORZNAB_PAGE_SIZE: usize = 100;
|
||||
|
||||
/// Bounded buffer between the streaming search producer task and the SSE-bound
|
||||
/// receiver stream. Backpressure: when the consumer (meta-share) is slow the
|
||||
/// producer's `send().await` parks rather than piling events in memory.
|
||||
pub(crate) const GATEWAY_STREAM_BUFFER: usize = 256;
|
||||
|
||||
/// Max results fetched **per indexer, per search** (depth cap). Bounds both the
|
||||
/// single-shot request for a non-paginating indexer and the offset-paging loop
|
||||
/// for one that supports pagination. Tunable; the merge across indexers + the
|
||||
/// caller's display `limit` truncate still apply on top. With today's 5 public
|
||||
/// trackers (all `limitsMax=100`, no pagination) the effective per-indexer
|
||||
/// fetch is 100, so 5×100 ≈ 500 merged — which is where this default lands.
|
||||
pub(crate) const TORZNAB_MAX_DEPTH: usize = 500;
|
||||
|
||||
/// Ceiling on how many records one torrent may fan out to when a pack is
|
||||
/// unpacked into per-file episodes (or range-inferred). A guard against a
|
||||
/// pathological 1000-file torrent flooding one search; well above any real
|
||||
/// season pack. Mirrors the tribler feeder's identical cap.
|
||||
pub(crate) const MAX_FANOUT_FILES: usize = 100;
|
||||
@@ -0,0 +1,269 @@
|
||||
//! Keyword-less catalog discovery branch.
|
||||
//!
|
||||
//! meta-share has no catalog/browse primitive, so a Stremio-style client
|
||||
//! (meta-watch) can't ask "what's popular" — it can only search by keyword.
|
||||
//! This branch answers a **keyword-less** gateway query whose intent is carried
|
||||
//! by structured filters, by calling TMDB's catalog endpoints. TMDB is the
|
||||
//! catalog oracle; the results are clean, ranked entries, each a distinct TMDB
|
||||
//! entity by construction (so the per-hit anchor-stamping that causes the
|
||||
//! wrong-poster bug on the indexer path simply can't happen here).
|
||||
//!
|
||||
//! ## Query dialect
|
||||
//!
|
||||
//! - **mode** (one of) — `popular:true` → `/{movie,tv}/popular`,
|
||||
//! `trending:true` → `/trending/{movie,tv}/week`,
|
||||
//! `top_rated:true` → `/{movie,tv}/top_rated`.
|
||||
//! - **kind** (required, also drives meta-share routing) — `contentKind:movie`
|
||||
//! → movies, `contentKind:episode` (or `tv`/`tvshow`) → TV.
|
||||
//! - **anime** (optional) — `anime:true` switches to `/discover/{movie,tv}`
|
||||
//! filtered to the TMDB *anime* keyword, sorted to match the mode. Useful when
|
||||
//! the only reachable indexer is an anime tracker (Nyaa): anime catalog titles
|
||||
//! actually have torrents, so the rows populate.
|
||||
//!
|
||||
//! ## Seeds, not files
|
||||
//!
|
||||
//! The records emitted here are **metadata-only seeds** — `title` + `tmdbid`
|
||||
//! (+ year) — *not* playable releases. There is no torznab fan-out, no poster
|
||||
//! fetch, and no `compute_outcomes`: meta-watch consumes a seed only to read its
|
||||
//! title/tmdbid, then issues its *own* anchored per-title search
|
||||
//! (`<title> fileType:video tmdbid:NNN`) to map a real file behind the card.
|
||||
//!
|
||||
//! ## Why the marker stamps are load-bearing
|
||||
//!
|
||||
//! Both the gateway's own dispatcher-side filter and meta-share's consumer-side
|
||||
//! `record_matches` re-apply the query's structured filters to every returned
|
||||
//! record, and a **missing field fails the match** (drop). So each seed must
|
||||
//! echo *every* discovery filter the query carried (`popular`/`trending`/
|
||||
//! `top_rated`, `contentKind`, and `anime` when present) or it is dropped at one
|
||||
//! of the two tiers.
|
||||
|
||||
use futures::channel::mpsc::Sender;
|
||||
use futures::SinkExt;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use meta_feeder_sdk::query::{GatewayQuery, GatewaySearchEvent};
|
||||
use crate::tmdb_budget::{Lease, TmdbBudget};
|
||||
use meta_feeder_sdk::types::DiscoveryRecord;
|
||||
|
||||
use crate::consts::TMDB_ENRICH_WAIT_DEADLINE_SECS;
|
||||
use crate::tmdb::{TmdbCall, TmdbClient, TmdbHit, TmdbKind};
|
||||
|
||||
/// TMDB keyword id for "anime" — the `anime:true` filter narrows `/discover` to
|
||||
/// it so an anime-only tracker actually has matching torrents.
|
||||
const TMDB_ANIME_KEYWORD: &str = "210024";
|
||||
|
||||
/// The catalog-list flavour a discovery query asks for. The marker filter that
|
||||
/// selected it (`popular`/`trending`/`top_rated`) is echoed onto every seed so
|
||||
/// the record survives `record_matches` on both tiers.
|
||||
#[derive(Clone, Copy)]
|
||||
enum DiscoveryMode {
|
||||
Popular,
|
||||
Trending,
|
||||
TopRated,
|
||||
}
|
||||
|
||||
impl DiscoveryMode {
|
||||
/// The query-filter key that selects this mode (and is stamped on seeds).
|
||||
fn marker(self) -> &'static str {
|
||||
match self {
|
||||
DiscoveryMode::Popular => "popular",
|
||||
DiscoveryMode::Trending => "trending",
|
||||
DiscoveryMode::TopRated => "top_rated",
|
||||
}
|
||||
}
|
||||
|
||||
/// `sort_by` value for the `/discover` (anime) path. `/discover` has no
|
||||
/// "trending", so trending approximates to popularity — the dedicated
|
||||
/// non-anime path still uses the real `/trending` endpoint.
|
||||
fn discover_sort(self) -> &'static str {
|
||||
match self {
|
||||
DiscoveryMode::TopRated => "vote_average.desc",
|
||||
DiscoveryMode::Popular | DiscoveryMode::Trending => "popularity.desc",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True when this is a keyword-less catalog-discovery query the branch should
|
||||
/// answer instead of the indexer fan-out: no free text and a truthy
|
||||
/// mode marker. A `contentKind` is still required to route at all (meta-share
|
||||
/// keys gateway routing on it) and to pick the movie-vs-TV endpoint.
|
||||
pub(crate) fn is_discovery_query(query: &GatewayQuery) -> bool {
|
||||
query.free_text.trim().is_empty() && discovery_mode(query).is_some()
|
||||
}
|
||||
|
||||
/// First truthy mode marker on the query, in precedence order.
|
||||
fn discovery_mode(query: &GatewayQuery) -> Option<DiscoveryMode> {
|
||||
for mode in [
|
||||
DiscoveryMode::Trending,
|
||||
DiscoveryMode::Popular,
|
||||
DiscoveryMode::TopRated,
|
||||
] {
|
||||
if filter_is_true(query, mode.marker()) {
|
||||
return Some(mode);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True iff `query.filters[key]` carries a truthy value.
|
||||
fn filter_is_true(query: &GatewayQuery, key: &str) -> bool {
|
||||
query
|
||||
.filters
|
||||
.get(key)
|
||||
.is_some_and(|v| v.iter().any(|s| s.eq_ignore_ascii_case("true")))
|
||||
}
|
||||
|
||||
/// Map the query's `contentKind` filter to a TMDB media kind.
|
||||
fn discovery_kind(query: &GatewayQuery) -> Option<TmdbKind> {
|
||||
let values = query.filters.get("contentKind")?;
|
||||
for v in values {
|
||||
match v.trim().to_ascii_lowercase().as_str() {
|
||||
"movie" => return Some(TmdbKind::Movie),
|
||||
"episode" | "tvshow" | "tv" => return Some(TmdbKind::Tv),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// TMDB returns this many results per catalog page; used to translate a desired
|
||||
/// seed count into a page count (`discover_records` walks pages until it has
|
||||
/// `max_results` hits).
|
||||
const TMDB_PAGE_SIZE: usize = 20;
|
||||
|
||||
/// Build the `path_and_query` for [`TmdbClient::discovery_list`] from the mode,
|
||||
/// kind, anime flag, and 1-based `page`.
|
||||
fn build_path_and_query(mode: DiscoveryMode, kind: TmdbKind, anime: bool, page: u32) -> String {
|
||||
let seg = match kind {
|
||||
TmdbKind::Movie => "movie",
|
||||
TmdbKind::Tv => "tv",
|
||||
};
|
||||
if anime {
|
||||
// `/discover` is the only endpoint that takes a keyword filter. Approx
|
||||
// top-rated with a vote-count floor so a 1-vote 10.0 doesn't win.
|
||||
let floor = match mode {
|
||||
DiscoveryMode::TopRated => "&vote_count.gte=200",
|
||||
_ => "",
|
||||
};
|
||||
format!(
|
||||
"discover/{seg}?with_keywords={TMDB_ANIME_KEYWORD}&sort_by={}&include_adult=false{floor}&page={page}",
|
||||
mode.discover_sort(),
|
||||
)
|
||||
} else {
|
||||
match mode {
|
||||
DiscoveryMode::Popular => format!("{seg}/popular?page={page}"),
|
||||
DiscoveryMode::TopRated => format!("{seg}/top_rated?page={page}"),
|
||||
DiscoveryMode::Trending => format!("trending/{seg}/week?page={page}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build one metadata-only seed record from a discovery hit. `record_id` is a
|
||||
/// synthetic, deterministic TMDB handle (`tmdb:movie:603`). Every discovery
|
||||
/// marker the query carried is echoed so the seed survives `record_matches`.
|
||||
fn build_seed_record(
|
||||
hit: &TmdbHit,
|
||||
kind: TmdbKind,
|
||||
mode: DiscoveryMode,
|
||||
anime: bool,
|
||||
) -> DiscoveryRecord {
|
||||
let (content_kind, seg) = match kind {
|
||||
TmdbKind::Movie => ("movie", "movie"),
|
||||
TmdbKind::Tv => ("episode", "tv"),
|
||||
};
|
||||
let mut fields = BTreeMap::new();
|
||||
// Load-bearing marker stamps (see module doc): the query's filters were the
|
||||
// mode marker, contentKind, and optionally anime.
|
||||
fields.insert(mode.marker().to_string(), "true".to_string());
|
||||
fields.insert("contentKind".to_string(), content_kind.to_string());
|
||||
fields.insert("fileType".to_string(), "video".to_string());
|
||||
if anime {
|
||||
fields.insert("anime".to_string(), "true".to_string());
|
||||
}
|
||||
// Data the consumer reads to seed its per-title search.
|
||||
fields.insert("title".to_string(), hit.title.clone());
|
||||
fields.insert("tmdbid".to_string(), hit.tmdbid.to_string());
|
||||
if let Some(y) = hit.year {
|
||||
fields.insert("movieYear".to_string(), y.to_string());
|
||||
}
|
||||
DiscoveryRecord {
|
||||
upstream_id: "prowlarr".to_string(),
|
||||
record_id: format!("tmdb:{seg}:{}", hit.tmdbid),
|
||||
fields,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a discovery query to up to `max_results` catalog seed records,
|
||||
/// walking TMDB pages (20/page) until the cap is reached or a page is empty.
|
||||
/// Each page is a separate budget-gated TMDB call; `[]` on no mode/contentKind,
|
||||
/// and an early break on the first rate-limit/miss (returning what we have).
|
||||
pub(crate) async fn discover_records(
|
||||
client: &TmdbClient,
|
||||
budget: Option<&Arc<TmdbBudget>>,
|
||||
query: &GatewayQuery,
|
||||
max_results: usize,
|
||||
) -> Vec<DiscoveryRecord> {
|
||||
let (Some(mode), Some(kind)) = (discovery_mode(query), discovery_kind(query)) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let anime = filter_is_true(query, "anime");
|
||||
// TMDB pages at TMDB_PAGE_SIZE; walk as many pages as needed to reach
|
||||
// `max_results` seeds (meta-watch asks for ~30, i.e. 2 pages). Each page is a
|
||||
// separate, budget-gated TMDB call.
|
||||
let pages = max_results.div_ceil(TMDB_PAGE_SIZE).max(1) as u32;
|
||||
let mut out: Vec<DiscoveryRecord> = Vec::new();
|
||||
for page in 1..=pages {
|
||||
if out.len() >= max_results {
|
||||
break;
|
||||
}
|
||||
if let Some(budget) = budget {
|
||||
if matches!(
|
||||
budget
|
||||
.acquire(Duration::from_secs(TMDB_ENRICH_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let path_and_query = build_path_and_query(mode, kind, anime, page);
|
||||
match client.discovery_list(kind, &path_and_query).await {
|
||||
TmdbCall::Hit(hits) => {
|
||||
if hits.is_empty() {
|
||||
break; // ran past the last catalog page
|
||||
}
|
||||
out.extend(hits.iter().map(|h| build_seed_record(h, kind, mode, anime)));
|
||||
}
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
if let Some(budget) = budget {
|
||||
budget.note_429(retry);
|
||||
}
|
||||
break;
|
||||
}
|
||||
TmdbCall::Miss => break,
|
||||
}
|
||||
}
|
||||
out.truncate(max_results);
|
||||
out
|
||||
}
|
||||
|
||||
/// Streaming variant: resolve the catalog page and push each seed as a `Base`
|
||||
/// event followed by `Done`. (A discovery list is a single HTTP call, so
|
||||
/// there's no incremental benefit — this just mirrors the streaming contract.)
|
||||
pub(crate) async fn discover_stream(
|
||||
client: Arc<TmdbClient>,
|
||||
budget: Option<Arc<TmdbBudget>>,
|
||||
query: GatewayQuery,
|
||||
max_results: usize,
|
||||
mut tx: Sender<GatewaySearchEvent>,
|
||||
) {
|
||||
let records = discover_records(&client, budget.as_ref(), &query, max_results).await;
|
||||
for record in records {
|
||||
if tx.send(GatewaySearchEvent::Base(record)).await.is_err() {
|
||||
return; // consumer gone
|
||||
}
|
||||
}
|
||||
let _ = tx.send(GatewaySearchEvent::Done).await;
|
||||
}
|
||||
@@ -0,0 +1,970 @@
|
||||
//! TMDB enrichment + poster store.
|
||||
//!
|
||||
//! Split out of the monolithic `torznab.rs` (pure file move; no behaviour change).
|
||||
|
||||
use crate::tmdb_budget::Lease;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::future::{BoxFuture, Shared};
|
||||
use futures::FutureExt;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::title::clean_torrent_title;
|
||||
use crate::tmdb::{
|
||||
principal_top_n, season_episode_bounds, SeasonEpisodeBounds, TmdbCall, TmdbClient,
|
||||
TmdbExternalIds, TmdbHit, TmdbKind, TmdbTvDetails,
|
||||
};
|
||||
use crate::tmdb_budget::TmdbBudget;
|
||||
use meta_feeder_sdk::cache::MidhashCache;
|
||||
use meta_feeder_sdk::query::GatewaySearchEvent;
|
||||
use meta_feeder_sdk::types::DiscoveryRecord;
|
||||
|
||||
/// Resolve a record's transient `posterPath` into a `poster_url` (the public
|
||||
/// TMDB poster CDN URL) in place, removing `posterPath`. In the feeder model the
|
||||
/// **gateway core** seeds `poster_url` into a content-addressed `poster` cid
|
||||
/// (same path as giphy/wikicommons previews) — the feeder never fetches or
|
||||
/// stores the poster bytes, so there is no meta-core poster record and no
|
||||
/// blockstore seed here. No-op when there's no `posterPath`.
|
||||
pub(crate) fn set_poster_url(record: &mut DiscoveryRecord, tmdb: &TmdbClient) {
|
||||
if let Some(path) = record.fields.remove("posterPath") {
|
||||
record
|
||||
.fields
|
||||
.insert("poster_url".to_string(), tmdb.poster_cdn_url(&path));
|
||||
}
|
||||
}
|
||||
|
||||
/// Long-lived TMDB + poster + subtitle-source state, split out of
|
||||
/// [`TorznabPlugin`]. Owns the enrichment config (TMDB client/budget/inflight,
|
||||
/// meta-core sink, bitswap blockstore, OpenSubtitles client + language set) and
|
||||
/// a clone of the shared cache/HTTP client. Builds the per-call [`TmdbEnricher`]
|
||||
/// and runs the search-time poster store. The plugin composes one of these and
|
||||
/// delegates its enrich methods here.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Enricher {
|
||||
pub(crate) cache: Option<MidhashCache>,
|
||||
pub(crate) tmdb: Option<Arc<TmdbClient>>,
|
||||
/// Process-global TMDB token budget. Self-initialised in [`Enricher::new`]
|
||||
/// (the feeder owns it now — the gateway's `set_tmdb_budget` injection is
|
||||
/// gone). `Option` is kept so `tmdb_enricher` / the call sites stay
|
||||
/// unchanged, but it is always `Some` in practice.
|
||||
pub(crate) tmdb_budget: Option<Arc<TmdbBudget>>,
|
||||
pub(crate) tmdb_inflight:
|
||||
Arc<Mutex<HashMap<String, Shared<BoxFuture<'static, Option<TmdbHit>>>>>>,
|
||||
}
|
||||
|
||||
impl Enricher {
|
||||
/// Construct the enricher; everything else is filled in by the setters /
|
||||
/// `configure`. The TMDB budget self-inits here (feeder-owned).
|
||||
pub(crate) fn new() -> Self {
|
||||
Self {
|
||||
cache: None,
|
||||
tmdb: None,
|
||||
tmdb_budget: Some(TmdbBudget::new(
|
||||
crate::tmdb_budget::DEFAULT_TMDB_REFILL_PER_SEC,
|
||||
crate::tmdb_budget::DEFAULT_TMDB_BURST,
|
||||
)),
|
||||
tmdb_inflight: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable TMDB enrichment with the given v4 bearer token. Empty /
|
||||
/// whitespace-only tokens leave it disabled (no-op). Idempotent.
|
||||
pub(crate) fn set_tmdb_token(&mut self, token: impl Into<String>) {
|
||||
let token = token.into();
|
||||
if token.trim().is_empty() {
|
||||
self.tmdb = None;
|
||||
} else {
|
||||
self.tmdb = Some(Arc::new(TmdbClient::new(token)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Test-only: inject a [`TmdbClient`] with mock base URLs.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_tmdb_client_for_test(&mut self, client: TmdbClient) {
|
||||
self.tmdb = Some(Arc::new(client));
|
||||
}
|
||||
|
||||
/// Build a per-call [`TmdbEnricher`] from the long-lived state. `None` when
|
||||
/// TMDB or the cache isn't configured (enrichment disabled). Cheap — every
|
||||
/// field cloned is an `Arc`/clone-cheap handle, and the single-flight map is
|
||||
/// shared (not copied) so coalescing spans calls.
|
||||
pub(crate) fn tmdb_enricher(&self) -> Option<TmdbEnricher> {
|
||||
Some(TmdbEnricher {
|
||||
client: self.tmdb.clone()?,
|
||||
cache: self.cache.clone()?,
|
||||
budget: self.tmdb_budget.clone(),
|
||||
inflight: self.tmdb_inflight.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// Bundles the shared state a TMDB enrichment pass needs: the client, the
|
||||
/// persistent redb cache, the optional process-global budget, and the
|
||||
/// single-flight map. Cheap to clone (everything inside is `Arc`/clone-cheap).
|
||||
/// Built per `handle_query` call by [`Enricher::tmdb_enricher`]; the
|
||||
/// single-flight map is shared, not copied, so concurrent identical lookups
|
||||
/// across many calls coalesce into one upstream request.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TmdbEnricher {
|
||||
pub(crate) client: Arc<TmdbClient>,
|
||||
pub(crate) cache: MidhashCache,
|
||||
pub(crate) budget: Option<Arc<TmdbBudget>>,
|
||||
pub(crate) inflight: Arc<Mutex<HashMap<String, Shared<BoxFuture<'static, Option<TmdbHit>>>>>>,
|
||||
}
|
||||
|
||||
impl TmdbEnricher {
|
||||
/// Cache → single-flight → budget-gated TMDB title search. Persistent
|
||||
/// cache hits and single-flight followers consume **no** budget token;
|
||||
/// only the leader future acquires one.
|
||||
pub(crate) async fn search_cached(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
title: &str,
|
||||
year: Option<u16>,
|
||||
) -> Option<TmdbHit> {
|
||||
let key = tmdb_search_key(kind, title, year);
|
||||
// 1. Persistent cache (positive hit JSON or the `"null"` negative
|
||||
// sentinel — both terminate without touching the budget).
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_search(&key) {
|
||||
return decode_cached_hit(&json);
|
||||
}
|
||||
// 2. Single-flight: the leader does the work; followers clone and
|
||||
// await the same shared future.
|
||||
let shared = {
|
||||
let mut map = self.inflight.lock().expect("tmdb inflight poisoned");
|
||||
if let Some(existing) = map.get(&key) {
|
||||
existing.clone()
|
||||
} else {
|
||||
let fut = self
|
||||
.clone()
|
||||
.search_leader(kind, title.to_string(), year, key.clone());
|
||||
let shared = fut.boxed().shared();
|
||||
map.insert(key.clone(), shared.clone());
|
||||
shared
|
||||
}
|
||||
};
|
||||
shared.await
|
||||
}
|
||||
|
||||
/// The single-flight leader: gate on the budget, call TMDB, persist the
|
||||
/// result, and de-register from the in-flight map. Owns a cloned
|
||||
/// [`TmdbEnricher`] so the future is `'static` for `Shared`.
|
||||
pub(crate) async fn search_leader(
|
||||
self,
|
||||
kind: TmdbKind,
|
||||
title: String,
|
||||
year: Option<u16>,
|
||||
key: String,
|
||||
) -> Option<TmdbHit> {
|
||||
if let Some(budget) = &self.budget {
|
||||
if matches!(
|
||||
budget
|
||||
.acquire(Duration::from_secs(TMDB_ENRICH_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
// Couldn't get a token in time — transient, so don't poison
|
||||
// the cache. De-register and surface a miss for this pass.
|
||||
self.inflight
|
||||
.lock()
|
||||
.expect("tmdb inflight poisoned")
|
||||
.remove(&key);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let (result, cache_write) = match self.client.search(kind, &title, year).await {
|
||||
TmdbCall::Hit(h) => {
|
||||
let json = serde_json::to_string(&h).ok();
|
||||
(Some(h), json)
|
||||
}
|
||||
// Confirmed miss: negative-cache it so the same title doesn't
|
||||
// re-burn a budget token on the next search.
|
||||
TmdbCall::Miss => (None, Some("null".to_string())),
|
||||
// Rate-limited: pause the whole budget, but DON'T cache (the
|
||||
// miss is transient — a later search should retry).
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
if let Some(budget) = &self.budget {
|
||||
budget.note_429(retry);
|
||||
}
|
||||
(None, None)
|
||||
}
|
||||
};
|
||||
if let Some(json) = cache_write {
|
||||
let _ = self.cache.put_tmdb_search(&key, &json);
|
||||
}
|
||||
self.inflight
|
||||
.lock()
|
||||
.expect("tmdb inflight poisoned")
|
||||
.remove(&key);
|
||||
result
|
||||
}
|
||||
|
||||
/// Budget-gate a single TMDB call, then resolve it: acquire a token
|
||||
/// (returning `None` on deadline), map `Hit(x)` → `Some(on_hit(x))`,
|
||||
/// `Miss` → `None`, and `RateLimited` → note the 429 and `None`. Each
|
||||
/// caller keeps its own persistent-cache fast path (the decode / self-heal
|
||||
/// rules differ per table) and does its cache write inside `on_hit`. This
|
||||
/// centralises the budget-acquire + 429 handling the by-id detail lookups
|
||||
/// all repeat. The single-flight title search ([`search_leader`]) keeps its
|
||||
/// own copy — it also negative-caches and de-registers from the in-flight
|
||||
/// map, which doesn't fit this shape.
|
||||
async fn budgeted_call<T, R>(
|
||||
&self,
|
||||
call: impl std::future::Future<Output = TmdbCall<T>>,
|
||||
on_hit: impl FnOnce(T) -> R,
|
||||
) -> Option<R> {
|
||||
if let Some(budget) = &self.budget {
|
||||
if matches!(
|
||||
budget
|
||||
.acquire(Duration::from_secs(TMDB_ENRICH_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
match call.await {
|
||||
TmdbCall::Hit(x) => Some(on_hit(x)),
|
||||
TmdbCall::Miss => None,
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
if let Some(budget) = &self.budget {
|
||||
budget.note_429(retry);
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache + budget-gated TMDB TV-details lookup (no single-flight — the
|
||||
/// season-bearing-episode path is lower volume and the redb cache catches
|
||||
/// most repeats). Cache hits consume no budget token.
|
||||
pub(crate) async fn tv_details_cached(&self, tmdbid: u64) -> Option<TmdbTvDetails> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_tvdetails(&key) {
|
||||
if let Ok(details) = serde_json::from_str::<TmdbTvDetails>(&json) {
|
||||
// Self-heal: an entry written before the display fields existed
|
||||
// (or by a bounds-only fetch) lacks `name`/`overview`/poster —
|
||||
// fall through to a fresh fetch so the anchored enrich path can
|
||||
// build a hit. Structurally-complete entries return as-is.
|
||||
if details.has_display() {
|
||||
return Some(details);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.budgeted_call(self.client.tv_details(tmdbid), |details| {
|
||||
if let Ok(json) = serde_json::to_string(&details) {
|
||||
let _ = self.cache.put_tmdb_tvdetails(&key, &json);
|
||||
}
|
||||
details
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cache + budget-gated TMDB `external_ids` lookup (`tvdb_id` / `imdb_id`
|
||||
/// for a known tmdbid). External ids are immutable, so a hit is cached
|
||||
/// forever. Cache hits consume no budget token.
|
||||
pub(crate) async fn external_ids_cached(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
tmdbid: u64,
|
||||
) -> Option<TmdbExternalIds> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_extids(&key) {
|
||||
return serde_json::from_str(&json).ok();
|
||||
}
|
||||
self.budgeted_call(self.client.external_ids(kind, tmdbid), |ids| {
|
||||
if let Ok(json) = serde_json::to_string(&ids) {
|
||||
let _ = self.cache.put_tmdb_extids(&key, &json);
|
||||
}
|
||||
ids
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cache + budget-gated TMDB `GET /3/movie/{id}` → [`TmdbHit`] (resolved by
|
||||
/// id). The anchored movie enrichment source. Cache hits consume no budget.
|
||||
pub(crate) async fn movie_hit_by_id_cached(&self, tmdbid: u64) -> Option<TmdbHit> {
|
||||
let key = tmdbid.to_string();
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_moviedetails(&key) {
|
||||
return decode_cached_hit(&json);
|
||||
}
|
||||
self.budgeted_call(self.client.movie_details(tmdbid), |details| {
|
||||
let hit = details.into_hit();
|
||||
if let Ok(json) = serde_json::to_string(&hit) {
|
||||
let _ = self.cache.put_tmdb_moviedetails(&key, &json);
|
||||
}
|
||||
hit
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a [`TmdbHit`] **by id** for an anchored record (known tmdbid),
|
||||
/// bypassing the fuzzy title search entirely. TV hits are derived from the
|
||||
/// cached TV-details payload (which doubles as the season-bounds source);
|
||||
/// movie hits from `GET /3/movie/{id}`. `None` when the lookup fails — the
|
||||
/// caller keeps the authoritative tmdbid and ships without the rest.
|
||||
pub(crate) async fn hit_by_id_cached(&self, kind: TmdbKind, tmdbid: u64) -> Option<TmdbHit> {
|
||||
match kind {
|
||||
TmdbKind::Tv => self.tv_details_cached(tmdbid).await?.as_hit(tmdbid),
|
||||
TmdbKind::Movie => self.movie_hit_by_id_cached(tmdbid).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cache + budget-gated principal `search/multi`: resolve a bare keyword to
|
||||
/// up to `n` confident `(tmdbid, kind)` anchors, most-popular first (or `[]`
|
||||
/// → caller keeps today's `q=` path). The full ranked list (up to
|
||||
/// [`CACHED_PRINCIPAL_DEPTH`]) is persisted and sliced to `n` on read, so a
|
||||
/// vague keyword doesn't re-burn a budget token every search and changing
|
||||
/// the top-N knob never needs a cache wipe. An empty list is negative-cached
|
||||
/// (the `"[]"` sentinel).
|
||||
pub(crate) async fn principal_top_n_cached(
|
||||
&self,
|
||||
query: &str,
|
||||
n: usize,
|
||||
) -> Vec<(u64, TmdbKind)> {
|
||||
let key = query.trim().to_lowercase();
|
||||
if key.is_empty() || n == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
if let Ok(Some(json)) = self.cache.get_tmdb_principal_topn(&key) {
|
||||
let mut list = decode_principal_list(&json);
|
||||
list.truncate(n);
|
||||
return list;
|
||||
}
|
||||
if let Some(budget) = &self.budget {
|
||||
if matches!(
|
||||
budget
|
||||
.acquire(Duration::from_secs(TMDB_ENRICH_WAIT_DEADLINE_SECS))
|
||||
.await,
|
||||
Lease::DeadlineExceeded
|
||||
) {
|
||||
return Vec::new(); // transient — don't poison the cache
|
||||
}
|
||||
}
|
||||
let ranked: Vec<(u64, TmdbKind)> = match self.client.search_multi(query).await {
|
||||
TmdbCall::Hit(hits) => principal_top_n(&hits, query, CACHED_PRINCIPAL_DEPTH)
|
||||
.into_iter()
|
||||
.filter_map(|h| h.kind().map(|k| (h.id, k)))
|
||||
.collect(),
|
||||
// The client folds a transient error (timeout / non-2xx) and a
|
||||
// genuine "no results" into the same `Miss`, so we can't tell them
|
||||
// apart here. Do NOT negative-cache it: a transient TMDB hiccup must
|
||||
// not disable multi-anchor for this keyword until the next cache
|
||||
// wipe (the bug that made a once-failed "black" search permanently
|
||||
// anchor-less). Retry on the next search instead.
|
||||
TmdbCall::Miss => return Vec::new(),
|
||||
TmdbCall::RateLimited(retry) => {
|
||||
if let Some(budget) = &self.budget {
|
||||
budget.note_429(retry);
|
||||
}
|
||||
return Vec::new(); // don't cache a rate-limit
|
||||
}
|
||||
};
|
||||
// Cache the resolved list — including an empty one, which here means
|
||||
// "TMDB responded but nothing passed the relevance gate" (a genuine
|
||||
// negative, safe to remember). Transient misses returned above, uncached.
|
||||
let _ = self
|
||||
.cache
|
||||
.put_tmdb_principal_topn(&key, &encode_principal_list(&ranked));
|
||||
let mut out = ranked;
|
||||
out.truncate(n);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON shape persisted in the principal-search caches.
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
struct PrincipalEntry {
|
||||
tmdbid: u64,
|
||||
kind: String,
|
||||
}
|
||||
|
||||
fn kind_str(kind: TmdbKind) -> &'static str {
|
||||
match kind {
|
||||
TmdbKind::Movie => "movie",
|
||||
TmdbKind::Tv => "tv",
|
||||
}
|
||||
}
|
||||
|
||||
fn kind_from_str(s: &str) -> Option<TmdbKind> {
|
||||
match s {
|
||||
"movie" => Some(TmdbKind::Movie),
|
||||
"tv" => Some(TmdbKind::Tv),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a ranked anchor list (most-popular first) as a JSON array; `"[]"`
|
||||
/// for empty (the negative-cache sentinel).
|
||||
fn encode_principal_list(list: &[(u64, TmdbKind)]) -> String {
|
||||
let entries: Vec<PrincipalEntry> = list
|
||||
.iter()
|
||||
.map(|(id, kind)| PrincipalEntry {
|
||||
tmdbid: *id,
|
||||
kind: kind_str(*kind).to_string(),
|
||||
})
|
||||
.collect();
|
||||
serde_json::to_string(&entries).unwrap_or_else(|_| "[]".to_string())
|
||||
}
|
||||
|
||||
/// Decode a ranked anchor list; unknown kinds are dropped, malformed JSON → [].
|
||||
fn decode_principal_list(json: &str) -> Vec<(u64, TmdbKind)> {
|
||||
serde_json::from_str::<Vec<PrincipalEntry>>(json)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter_map(|e| kind_from_str(&e.kind).map(|k| (e.tmdbid, k)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Stable lookup key for the TMDB-search cache + single-flight map:
|
||||
/// `"<m|t>\x01<cleaned-title>\x01<year-or-empty>"`.
|
||||
pub(crate) fn tmdb_search_key(kind: TmdbKind, title: &str, year: Option<u16>) -> String {
|
||||
let k = match kind {
|
||||
TmdbKind::Movie => 'm',
|
||||
TmdbKind::Tv => 't',
|
||||
};
|
||||
let y = year.map(|y| y.to_string()).unwrap_or_default();
|
||||
format!("{k}\u{1}{title}\u{1}{y}")
|
||||
}
|
||||
|
||||
/// Decode a cached TMDB-search value: the `"null"` sentinel → confirmed miss
|
||||
/// (`None`), anything else → the encoded [`TmdbHit`].
|
||||
pub(crate) fn decode_cached_hit(json: &str) -> Option<TmdbHit> {
|
||||
if json == "null" {
|
||||
return None;
|
||||
}
|
||||
serde_json::from_str(json).ok()
|
||||
}
|
||||
|
||||
/// Field-level enrichment delta, or a verdict to drop/skip the record.
|
||||
/// Computed without mutating the base record so the streaming path can emit
|
||||
/// it as a [`GatewaySearchEvent::EnrichPatch`] / [`GatewaySearchEvent::Drop`].
|
||||
pub(crate) enum EnrichOutcome {
|
||||
/// Apply `set` (insert/overwrite) and `remove` (delete) to the base record.
|
||||
Patch {
|
||||
set: BTreeMap<String, String>,
|
||||
remove: Vec<String>,
|
||||
},
|
||||
/// A TV match whose title-parsed season/episode is positively out of
|
||||
/// TMDB's bounds — retract the record entirely (mirrors the buffered
|
||||
/// path's `retain` drop).
|
||||
Drop,
|
||||
/// Not a TV/movie record, no usable title, or no TMDB match — leave the
|
||||
/// base record untouched, emit no patch.
|
||||
Noop,
|
||||
}
|
||||
|
||||
/// Best-effort TMDB enrichment for a single record, computed as a delta.
|
||||
/// No-op if the record's contentKind isn't a known TV/movie kind, if title
|
||||
/// cleaning fails to produce a non-empty query, or if TMDB returns no hits.
|
||||
/// Routes every TMDB call through `enricher` (cache + single-flight + budget).
|
||||
///
|
||||
/// When the real enrichment is a [`EnrichOutcome::Noop`] and the test-only
|
||||
/// [`crate::consts::ENV_STUB_UNMATCHED`] flag is set, this falls back to
|
||||
/// [`stub_unmatched_patch`] — fabricated metadata so an unmatched record
|
||||
/// (e.g. an adult release tagged `categories/XXX` that never matches TMDB)
|
||||
/// still surfaces in meta-watch for filter validation. Off by default.
|
||||
pub(crate) async fn compute_enrichment(
|
||||
record: &DiscoveryRecord,
|
||||
enricher: &TmdbEnricher,
|
||||
) -> EnrichOutcome {
|
||||
match compute_enrichment_inner(record, enricher).await {
|
||||
EnrichOutcome::Noop => stub_unmatched_patch(record).unwrap_or(EnrichOutcome::Noop),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// Is the test-only unmatched-stub flag enabled? Read once. Exposed to
|
||||
/// `xml.rs` so `into_discovery_record` can stamp a base-record `fileType`
|
||||
/// for kind-less (e.g. `categories/XXX`) records — without it the
|
||||
/// `fileType:video` routing/base filter drops them before this enrichment
|
||||
/// stub runs (the stub only adds poster/description; see `build_stub_patch`).
|
||||
pub(crate) fn stub_unmatched_enabled() -> bool {
|
||||
static FLAG: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*FLAG.get_or_init(|| {
|
||||
std::env::var(crate::consts::ENV_STUB_UNMATCHED)
|
||||
.map(|v| {
|
||||
matches!(
|
||||
v.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
|
||||
/// **Test-only.** Fabricate the minimum metadata an unmatched record needs to
|
||||
/// pass meta-watch's quality gate (`is_video && has_poster && has_description`,
|
||||
/// see `meta-watch/src/gate.rs`), so it surfaces in the UI for adult-filter
|
||||
/// validation. Returns `None` unless [`stub_unmatched_enabled`] and the record
|
||||
/// has a title. Only fills what's missing — a `5xxx` anime miss keeps its
|
||||
/// `episode` kind and just gains a poster/description; an `XXX` (`6xxx`) record
|
||||
/// with no kind at all gains video kind too. The `poster` is a non-resolvable
|
||||
/// marker (the tile renders broken — card visibility is what the test toggles),
|
||||
/// and the canonical `categories/XXX` tag is already on the record from
|
||||
/// `xml.rs`, so the consumer-side filter keys on it unchanged.
|
||||
fn stub_unmatched_patch(record: &DiscoveryRecord) -> Option<EnrichOutcome> {
|
||||
if !stub_unmatched_enabled() {
|
||||
return None;
|
||||
}
|
||||
build_stub_patch(&record.fields)
|
||||
}
|
||||
|
||||
/// Pure stub-patch builder (env-independent, for unit tests). Fills only the
|
||||
/// gate fields that are missing. Returns `None` if there's no title (can't
|
||||
/// render a card) or nothing needs fabricating (already passes the gate).
|
||||
fn build_stub_patch(f: &BTreeMap<String, String>) -> Option<EnrichOutcome> {
|
||||
if !f
|
||||
.get("title")
|
||||
.map(|t| !t.trim().is_empty())
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let is_video = f.get("fileType").map(|v| v.eq_ignore_ascii_case("video")) == Some(true)
|
||||
|| matches!(
|
||||
f.get("videoType")
|
||||
.map(|v| v.to_ascii_lowercase())
|
||||
.as_deref(),
|
||||
Some("movie") | Some("tvshow") | Some("episode")
|
||||
);
|
||||
let has_poster = f
|
||||
.get("poster")
|
||||
.map(|p| !p.trim().is_empty())
|
||||
.unwrap_or(false);
|
||||
let has_desc = f
|
||||
.get("description")
|
||||
.map(|d| !d.trim().is_empty())
|
||||
.unwrap_or(false)
|
||||
|| f.get("plot/eng")
|
||||
.map(|d| !d.trim().is_empty())
|
||||
.unwrap_or(false);
|
||||
|
||||
let mut set: BTreeMap<String, String> = BTreeMap::new();
|
||||
if !is_video {
|
||||
set.insert("fileType".to_string(), "video".to_string());
|
||||
set.insert("videoType".to_string(), "movie".to_string());
|
||||
set.insert("contentKind".to_string(), "movie".to_string());
|
||||
}
|
||||
if !has_poster {
|
||||
// Non-resolvable marker — the image 404s (broken tile), but the card
|
||||
// renders, which is all the on/off filter test needs.
|
||||
set.insert("poster".to_string(), "stub-no-tmdb-poster".to_string());
|
||||
}
|
||||
if !has_desc {
|
||||
set.insert(
|
||||
"description".to_string(),
|
||||
"[STUB] Simulated unmatched/polluted record (no TMDB match) — for adult-filter testing.".to_string(),
|
||||
);
|
||||
}
|
||||
if set.is_empty() {
|
||||
// Already passes the gate on its own — nothing to fabricate.
|
||||
return None;
|
||||
}
|
||||
Some(EnrichOutcome::Patch {
|
||||
set,
|
||||
remove: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod stub_tests {
|
||||
use super::*;
|
||||
|
||||
fn fields(pairs: &[(&str, &str)]) -> BTreeMap<String, String> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn patch_set(outcome: Option<EnrichOutcome>) -> BTreeMap<String, String> {
|
||||
match outcome {
|
||||
Some(EnrichOutcome::Patch { set, .. }) => set,
|
||||
_ => panic!("expected a stub patch"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xxx_record_with_no_kind_gets_full_video_stub() {
|
||||
// A pure-XXX record (no contentKind/fileType, just a title + category)
|
||||
// gets video kind + poster + description so it passes meta-watch.
|
||||
let set = patch_set(build_stub_patch(&fields(&[
|
||||
("title", "Some Release"),
|
||||
("categories/XXX", "true"),
|
||||
])));
|
||||
assert_eq!(set.get("fileType").map(String::as_str), Some("video"));
|
||||
assert_eq!(set.get("videoType").map(String::as_str), Some("movie"));
|
||||
assert!(set.contains_key("poster"));
|
||||
assert!(set.contains_key("description"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_video_kind_and_desc_are_preserved() {
|
||||
// A 5xxx anime miss already carries episode kind; only the missing
|
||||
// poster is fabricated — kind and description are left untouched.
|
||||
let set = patch_set(build_stub_patch(&fields(&[
|
||||
("title", "Anime S01E01"),
|
||||
("fileType", "video"),
|
||||
("videoType", "tvshow"),
|
||||
("contentKind", "episode"),
|
||||
("plot/eng", "real synopsis"),
|
||||
])));
|
||||
assert!(!set.contains_key("fileType"));
|
||||
assert!(!set.contains_key("videoType"));
|
||||
assert!(!set.contains_key("description"));
|
||||
assert_eq!(
|
||||
set.get("poster").map(String::as_str),
|
||||
Some("stub-no-tmdb-poster")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_title_or_nothing_missing_returns_none() {
|
||||
// No title ⇒ can't build a card.
|
||||
assert!(build_stub_patch(&fields(&[("categories/XXX", "true")])).is_none());
|
||||
// Already passes the gate ⇒ nothing to fabricate.
|
||||
assert!(build_stub_patch(&fields(&[
|
||||
("title", "x"),
|
||||
("fileType", "video"),
|
||||
("poster", "real-cid"),
|
||||
("description", "real"),
|
||||
]))
|
||||
.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort TMDB enrichment for a single record, computed as a delta.
|
||||
/// No-op if the record's contentKind isn't a known TV/movie kind, if title
|
||||
/// cleaning fails to produce a non-empty query, or if TMDB returns no hits.
|
||||
/// Routes every TMDB call through `enricher` (cache + single-flight + budget).
|
||||
///
|
||||
/// Two stages: [`resolve_enrichment_hit`] picks the canonical TMDB entry (by
|
||||
/// anchored id, else fuzzy title search), then [`build_enrichment_patch`]
|
||||
/// turns that hit into the field delta (and runs the TV season/episode
|
||||
/// bounds-check that can drop the record).
|
||||
async fn compute_enrichment_inner(
|
||||
record: &DiscoveryRecord,
|
||||
enricher: &TmdbEnricher,
|
||||
) -> EnrichOutcome {
|
||||
// Only enrich TV/movie records.
|
||||
let kind = match record.fields.get("contentKind").map(String::as_str) {
|
||||
Some("movie") => TmdbKind::Movie,
|
||||
Some("episode") => TmdbKind::Tv,
|
||||
_ => return EnrichOutcome::Noop,
|
||||
};
|
||||
let raw_title = record
|
||||
.fields
|
||||
.get("title")
|
||||
.map(|t| t.trim().to_string())
|
||||
.filter(|t| !t.is_empty());
|
||||
|
||||
let (hit, matched_kind) = match resolve_enrichment_hit(record, enricher, kind, &raw_title).await
|
||||
{
|
||||
Some(resolved) => resolved,
|
||||
None => return EnrichOutcome::Noop,
|
||||
};
|
||||
build_enrichment_patch(record, enricher, hit, matched_kind, raw_title).await
|
||||
}
|
||||
|
||||
/// Pick the canonical TMDB entry for `record`. Prefers the **anchored** path —
|
||||
/// an authoritative `tmdbid` stamped on the Base by the structural
|
||||
/// tvsearch/movie fan-out or principal-search — resolving by id so a fuzzy
|
||||
/// match can't overwrite the anchor. Falls back to a **fuzzy** title search
|
||||
/// (with a movie↔tv kind fallback, since torznab categories mislabel kinds).
|
||||
/// `None` ⇒ the caller should no-op.
|
||||
async fn resolve_enrichment_hit(
|
||||
record: &DiscoveryRecord,
|
||||
enricher: &TmdbEnricher,
|
||||
kind: TmdbKind,
|
||||
raw_title: &Option<String>,
|
||||
) -> Option<(TmdbHit, TmdbKind)> {
|
||||
let anchored_id = record
|
||||
.fields
|
||||
.get("tmdbid")
|
||||
.and_then(|s| s.trim().parse::<u64>().ok());
|
||||
if let Some(id) = anchored_id {
|
||||
// Anchored: resolve by id. On lookup failure we keep the stamped
|
||||
// tmdbid and skip the rest (no-op).
|
||||
return enricher.hit_by_id_cached(kind, id).await.map(|h| (h, kind));
|
||||
}
|
||||
|
||||
let raw_title = raw_title.as_ref()?;
|
||||
let cleaned = clean_torrent_title(raw_title);
|
||||
if cleaned.title.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// Kind fallback: torznab/newznab categories mislabel films as episodes
|
||||
// (e.g. "CODE: White" arrives as cat 5070) and vice-versa. If the
|
||||
// primary endpoint misses, try the other before giving up.
|
||||
let other = match kind {
|
||||
TmdbKind::Tv => TmdbKind::Movie,
|
||||
TmdbKind::Movie => TmdbKind::Tv,
|
||||
};
|
||||
if let Some(h) = enricher
|
||||
.search_cached(kind, &cleaned.title, cleaned.year)
|
||||
.await
|
||||
{
|
||||
return Some((h, kind));
|
||||
}
|
||||
enricher
|
||||
.search_cached(other, &cleaned.title, cleaned.year)
|
||||
.await
|
||||
.map(|h| (h, other))
|
||||
}
|
||||
|
||||
/// Turn a resolved TMDB `hit` into the field delta: title/synopsis/year/tmdbid,
|
||||
/// content-kind reconciliation (movie clears bogus season/episode; tv normalises
|
||||
/// to `episode` and bounds-checks the parsed season/episode — an out-of-bounds
|
||||
/// verdict drops the record), and the transient `posterPath` the poster store
|
||||
/// consumes. `raw_title` is preserved as `originalTitle`.
|
||||
async fn build_enrichment_patch(
|
||||
record: &DiscoveryRecord,
|
||||
enricher: &TmdbEnricher,
|
||||
hit: TmdbHit,
|
||||
matched_kind: TmdbKind,
|
||||
raw_title: Option<String>,
|
||||
) -> EnrichOutcome {
|
||||
let mut set: BTreeMap<String, String> = BTreeMap::new();
|
||||
let mut remove: Vec<String> = Vec::new();
|
||||
set.insert("title".to_string(), hit.title.clone());
|
||||
// Preserve the raw tracker release title (quality tags / release-group
|
||||
// info) in `originalTitle`. This is the canonical home now that the
|
||||
// bespoke `releaseTitle` key is retired — reuse-before-invent
|
||||
// (METADATA_KEYS.md rule #1, §14.10 title sprawl).
|
||||
if let Some(raw_title) = raw_title {
|
||||
set.insert("originalTitle".to_string(), raw_title);
|
||||
}
|
||||
// TMDB's original-language title (e.g. "ナルト") is a genuine localized
|
||||
// title — file it under the namespaced `titles/{lang3}` (METADATA_KEYS.md
|
||||
// §3), keyed by TMDB's `original_language`, instead of clobbering
|
||||
// `originalTitle`. Skipped when the language is outside the common 639-1
|
||||
// set or the title is blank (originalTitle still carries the raw title).
|
||||
if let Some(orig) = hit.original_title.clone() {
|
||||
if !orig.trim().is_empty() {
|
||||
if let Some(lang3) = hit.original_lang3() {
|
||||
set.insert(format!("titles/{lang3}"), orig);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(o) = hit.overview.clone() {
|
||||
if !o.trim().is_empty() {
|
||||
// Canonical synopsis field is the language-namespaced
|
||||
// `description/{lang3}` (METADATA_KEYS.md §14.6) — the
|
||||
// cross-content-kind synopsis, and the form meta-share's free-text
|
||||
// tokenizer actually indexes (`description/*` prefix, §12). TMDB
|
||||
// search responses default to en-US (the client sends no
|
||||
// `language` param), so the overview is English prose → `eng`. The
|
||||
// flat `overview`/`description` keys are neither in the registry
|
||||
// nor indexed, so they'd be invisible to cohort search.
|
||||
set.insert("description/eng".to_string(), o);
|
||||
}
|
||||
}
|
||||
if let Some(y) = hit.year {
|
||||
set.insert("movieYear".to_string(), y.to_string());
|
||||
}
|
||||
set.insert("tmdbid".to_string(), hit.tmdbid.to_string());
|
||||
// Reconcile content kind against what TMDB actually matched. A movie
|
||||
// match clears the bogus season/episode; a TV match normalizes
|
||||
// contentKind to "episode".
|
||||
match matched_kind {
|
||||
TmdbKind::Movie => {
|
||||
set.insert("contentKind".to_string(), "movie".to_string());
|
||||
// Co-write the narrower `videoType` facet (METADATA_KEYS.md §14.11
|
||||
// — contentKind=movie ↔ videoType=movie). Overwrites any
|
||||
// category-derived videoType from `into_discovery_record` with the
|
||||
// TMDB-reconciled kind.
|
||||
set.insert("videoType".to_string(), "movie".to_string());
|
||||
set.insert("fileType".to_string(), "video".to_string());
|
||||
remove.push("season".to_string());
|
||||
remove.push("episode".to_string());
|
||||
}
|
||||
TmdbKind::Tv => {
|
||||
set.insert("contentKind".to_string(), "episode".to_string());
|
||||
// contentKind=episode ↔ videoType=tvshow (METADATA_KEYS.md §14.11).
|
||||
set.insert("videoType".to_string(), "tvshow".to_string());
|
||||
// Season/episode boundary validation. Only a parsed `season`
|
||||
// triggers a details lookup, then [`season_episode_bounds`] grades
|
||||
// the parse against TMDB's real structure:
|
||||
// - SeasonOverflow (the `S2`-of-a-1-TMDB-season anime cour case):
|
||||
// the TMDB match is sound, only the season number is a
|
||||
// fansub-numbering artifact — keep the record but strip the
|
||||
// misleading `season` so it doesn't claim a season TMDB lacks.
|
||||
// - Contradiction (negative season, or episode past a matched
|
||||
// season's count): the parse is wrong — drop the whole record
|
||||
// so meta-core never ingests it.
|
||||
let season = record
|
||||
.fields
|
||||
.get("season")
|
||||
.and_then(|s| s.trim().parse::<i64>().ok());
|
||||
if season.is_some() {
|
||||
let episode = record
|
||||
.fields
|
||||
.get("episode")
|
||||
.and_then(|e| e.trim().parse::<i64>().ok());
|
||||
if let Some(details) = enricher.tv_details_cached(hit.tmdbid).await {
|
||||
match season_episode_bounds(&details, season, episode) {
|
||||
SeasonEpisodeBounds::Ok => {}
|
||||
SeasonEpisodeBounds::SeasonOverflow => {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdbid = hit.tmdbid,
|
||||
season = ?season,
|
||||
episode = ?episode,
|
||||
number_of_seasons = details.number_of_seasons,
|
||||
record_id = %record.record_id,
|
||||
"title-parsed season past TMDB bounds (cour-as-season); stripping season, keeping record"
|
||||
);
|
||||
remove.push("season".to_string());
|
||||
}
|
||||
SeasonEpisodeBounds::Contradiction => {
|
||||
// The parsed season/episode contradicts TMDB's
|
||||
// structure — e.g. an explicit "S01E25" on a show
|
||||
// whose season 1 has 7 episodes. That is NOT absolute
|
||||
// numbering (absolute-numbered releases carry no
|
||||
// season token and are left season-less upstream, so
|
||||
// they never reach here); it's a mismatched or
|
||||
// garbled record, so drop it rather than surfacing a
|
||||
// phantom episode. We deliberately no longer try to
|
||||
// "rescue" it by remapping the episode onto a TMDB
|
||||
// season — a season is only ever asserted from a
|
||||
// trusted explicit token, never inferred.
|
||||
warn!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdbid = hit.tmdbid,
|
||||
season = ?season,
|
||||
episode = ?episode,
|
||||
number_of_seasons = details.number_of_seasons,
|
||||
record_id = %record.record_id,
|
||||
"title-parsed season/episode contradicts TMDB; dropping record"
|
||||
);
|
||||
return EnrichOutcome::Drop;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(p) = hit.poster_path.clone() {
|
||||
// Transient input only — `store_poster_inner` reads it to fetch the
|
||||
// bytes, sets `poster=<cid>`, and removes it before the record ships.
|
||||
let normalised = p.trim_start_matches('/').to_string();
|
||||
set.insert("posterPath".to_string(), normalised);
|
||||
}
|
||||
EnrichOutcome::Patch { set, remove }
|
||||
}
|
||||
|
||||
/// Buffered-path enrichment: apply [`compute_enrichment`]'s delta to `record`
|
||||
/// in place. Returns `false` when the record should be dropped (out-of-bounds
|
||||
/// season/episode).
|
||||
pub(crate) async fn enrich_record_with_tmdb(
|
||||
record: &mut DiscoveryRecord,
|
||||
enricher: &TmdbEnricher,
|
||||
) -> bool {
|
||||
match compute_enrichment(record, enricher).await {
|
||||
EnrichOutcome::Drop => false,
|
||||
EnrichOutcome::Noop => true,
|
||||
EnrichOutcome::Patch { set, remove } => {
|
||||
for k in &remove {
|
||||
record.fields.remove(k);
|
||||
}
|
||||
for (k, v) in set {
|
||||
record.fields.insert(k, v);
|
||||
}
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Streaming-path enrichment for one base record: compute the TMDB delta,
|
||||
/// store the poster (off-budget), write the enriched bibrec sidecar, and
|
||||
/// return the [`GatewaySearchEvent`]s the consumer should apply on top of the
|
||||
/// already-emitted `Base`. Owns its inputs so the enclosing stream is
|
||||
/// `'static` (dropping the stream cancels in-flight TMDB work and frees the
|
||||
/// budget). Mirrors the buffered `handle_query` enrich → poster → bibrec
|
||||
/// phases, minus the cross-record `retain` (a drop is signalled per-record via
|
||||
/// a `Drop` event instead).
|
||||
pub(crate) async fn enrich_one_streaming(
|
||||
record: DiscoveryRecord,
|
||||
enricher: Option<TmdbEnricher>,
|
||||
cache: Option<MidhashCache>,
|
||||
) -> Vec<GatewaySearchEvent> {
|
||||
let record_id = record.record_id.clone();
|
||||
let mut working = record;
|
||||
let mut events: Vec<GatewaySearchEvent> = Vec::new();
|
||||
|
||||
// 1. TMDB metadata enrichment (budget-gated inside the enricher).
|
||||
if let Some(enricher) = &enricher {
|
||||
match compute_enrichment(&working, enricher).await {
|
||||
EnrichOutcome::Drop => {
|
||||
// The Base already shipped; retract it. Don't poster-store or
|
||||
// bibrec-cache the bogus record (matches the buffered retain).
|
||||
events.push(GatewaySearchEvent::Drop { record_id });
|
||||
return events;
|
||||
}
|
||||
EnrichOutcome::Noop => {}
|
||||
EnrichOutcome::Patch { set, remove } => {
|
||||
for k in &remove {
|
||||
working.fields.remove(k);
|
||||
}
|
||||
for (k, v) in &set {
|
||||
working.fields.insert(k.clone(), v.clone());
|
||||
}
|
||||
// `posterPath` is an internal transient — `store_poster_inner`
|
||||
// consumes it and emits `poster` instead. Never expose it.
|
||||
let mut wire_set = set;
|
||||
wire_set.remove("posterPath");
|
||||
if !wire_set.is_empty() || !remove.is_empty() {
|
||||
events.push(GatewaySearchEvent::EnrichPatch {
|
||||
record_id: record_id.clone(),
|
||||
set: wire_set,
|
||||
remove,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Poster: resolve the TMDB CDN url and emit it as `poster_url` (off the
|
||||
// TMDB API budget — different CDN host). The gateway core seeds it into a
|
||||
// content-addressed `poster` cid; the feeder no longer fetches or stores
|
||||
// the bytes. `poster_url` IS persisted into `working.fields` so it rides
|
||||
// bibrec into `compute_outcomes`, where the core seeds it onto the stored
|
||||
// video record too.
|
||||
if let Some(enricher) = &enricher {
|
||||
if working.fields.contains_key("posterPath") {
|
||||
set_poster_url(&mut working, &enricher.client);
|
||||
if let Some(url) = working.fields.get("poster_url").cloned() {
|
||||
let mut set = BTreeMap::new();
|
||||
set.insert("poster_url".to_string(), url);
|
||||
events.push(GatewaySearchEvent::EnrichPatch {
|
||||
record_id: record_id.clone(),
|
||||
set,
|
||||
remove: vec!["posterPath".to_string()],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. bibrec sidecar (enriched fields) — same as the buffered path.
|
||||
if let Some(cache) = &cache {
|
||||
if let Err(e) = cache.put_bibrec(&working.record_id, &working.fields) {
|
||||
warn!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
record_id = working.record_id,
|
||||
error = %e,
|
||||
"bibrec cache put failed (non-fatal; compute_outcomes will NotFound)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
events
|
||||
}
|
||||
|
||||
use crate::consts::*;
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Title-tag parsing moved to the shared `meta_feeder_sdk::filename_meta`
|
||||
//! (de-duplicated — was copied verbatim in tribler + torznab feeders). This
|
||||
//! re-export keeps the existing `crate::filename_meta::extract_*` call sites
|
||||
//! working unchanged. New code can use `meta_feeder_sdk::filename_meta` directly.
|
||||
pub use meta_feeder_sdk::filename_meta::*;
|
||||
@@ -0,0 +1,51 @@
|
||||
//! `tribler-feeder` library surface. The binary (`main.rs`) wraps the Tribler
|
||||
//! plugin in [`meta_feeder_sdk::serve_feeders`].
|
||||
//!
|
||||
//! This crate was split out of `meta-feeder-torrent` (the unified torrent
|
||||
//! feeder) so tribler is a **single-plugin** feeder — its dashboard config
|
||||
//! panel resolves to the tribler schema instead of colliding with prowlarr on a
|
||||
//! shared feeder's single `/config` surface.
|
||||
//!
|
||||
//! It carries a **copy** of the source-agnostic "torrent-core" TMDB
|
||||
//! catalog-discovery stack (`tmdb`, `tmdb_budget`, `discovery`, `enrich`,
|
||||
//! `title`, `consts`, `filename_meta`). That stack cannot live in a container
|
||||
//! plugin — a container plugin enriches a known CID, it can't answer "what's
|
||||
//! popular"; the browse/catalog primitive is inherently in-process and shared by
|
||||
//! every torrent source. The librqbit BitTorrent full-file fetch (`bt.rs` in
|
||||
//! meta-feeder-torrent) is **not** copied here: the tribler source enumerates a
|
||||
//! torrent's files from the Tribler core's metainfo response (`tribler::metainfo`),
|
||||
//! never via librqbit.
|
||||
|
||||
// The copied torrent-core modules carry the same clippy lints allowed in the
|
||||
// meta-feeder-torrent origin, so this crate meets a `-D warnings` bar without
|
||||
// diverging the copies from their twin. `dead_code` is allowed because the
|
||||
// copied TMDB stack is source-agnostic: the tribler source drives catalog
|
||||
// discovery + streaming enrichment, but not the indexer-result reconciliation
|
||||
// helpers (norm_tokens / principal_top_n / episode_in_season / the TMDB
|
||||
// multi-search DTOs) that only the torznab source in meta-feeder-torrent calls.
|
||||
// Keeping them verbatim (rather than pruning) keeps the copies diff-clean
|
||||
// against their twin.
|
||||
#![allow(
|
||||
dead_code,
|
||||
clippy::type_complexity,
|
||||
clippy::unnecessary_get_then_check,
|
||||
clippy::needless_borrow,
|
||||
clippy::needless_borrows_for_generic_args,
|
||||
clippy::manual_find,
|
||||
clippy::if_same_then_else,
|
||||
clippy::items_after_test_module
|
||||
)]
|
||||
|
||||
// Source-agnostic torrent pipeline (copied "torrent-core"): TMDB client +
|
||||
// budget, keyword-less catalog discovery, release-title cleaning, and the
|
||||
// in-Rust TMDB enrichment reconciliation used on the discovery/search stream.
|
||||
pub mod consts;
|
||||
pub mod discovery;
|
||||
pub mod enrich;
|
||||
pub mod filename_meta;
|
||||
pub mod title;
|
||||
pub mod tmdb;
|
||||
pub mod tmdb_budget;
|
||||
|
||||
// The tribler source itself (IPv8-overlay client of a headless Tribler core).
|
||||
pub mod tribler;
|
||||
@@ -0,0 +1,49 @@
|
||||
//! `tribler-feeder` — single-source BitTorrent feeder sidecar for the
|
||||
//! decentralized Tribler (IPv8 overlay) network.
|
||||
//!
|
||||
//! One plugin (`tribler`) served through [`meta_feeder_sdk::serve_feeders`]. A
|
||||
//! thin client of a headless Tribler core (reached at `TRIBLER_SIDECAR_URL`); it
|
||||
//! finds records over the Tribler swarm and enriches them via the copied
|
||||
//! torrent-core TMDB catalog-discovery stack. Split out of `meta-feeder-torrent`
|
||||
//! so its dashboard config panel resolves to the tribler schema.
|
||||
//!
|
||||
//! Env (shared): `META_FEEDER_HTTP_LISTEN` (default `0.0.0.0:8080`),
|
||||
//! `META_FEEDER_STATE_DIR` (default `/data/meta-feeder`), `META_CORE_URL` (D4
|
||||
//! self-publish), `RUST_LOG`.
|
||||
//! Env (tribler source): `TRIBLER_SIDECAR_URL`, `TRIBLER_API_KEY`,
|
||||
//! `TMDB_TOKEN`/`TMDB_LANGUAGE`, plus the enrichment-plugin URLs
|
||||
//! (`FILENAME_PARSER_URL`, `TMDB_PLUGIN_URL`, `META_FEEDER_CALLBACK_URL`).
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use meta_feeder_sdk::plugin::FeederPlugin;
|
||||
use meta_feeder_sdk::serve_feeders;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use tribler_feeder::tribler::{TriblerConfigFile, TriblerPlugin};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
|
||||
.init();
|
||||
|
||||
let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string())
|
||||
.parse()?;
|
||||
let state_dir =
|
||||
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
||||
|
||||
// Env is the first-boot seed; the persisted config.json (under the
|
||||
// per-source cache dir) wins once saved through the dashboard. The tribler
|
||||
// source builds its enrichment driver in configure().
|
||||
let mut tr = TriblerPlugin::new();
|
||||
tr.set_seed_config(TriblerConfigFile::from_env());
|
||||
|
||||
let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(tr)];
|
||||
|
||||
info!(target: "meta-feeder", loaded = plugins.len(), "tribler feeder starting");
|
||||
|
||||
serve_feeders(plugins, state_dir, listen).await
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
//! Pure torrent-title-cleaning logic.
|
||||
//!
|
||||
//! Split out of the monolithic `torznab.rs` (pure file move; no behaviour change).
|
||||
|
||||
use crate::filename_meta;
|
||||
use crate::tmdb::TmdbKind;
|
||||
|
||||
// -- Title cleaner -----------------------------------------------------------
|
||||
//
|
||||
// Torrent titles encode a lot of release-metadata noise around the
|
||||
// actual movie/show name: `[Group] Show - 117 [1080p].mkv` is typical.
|
||||
// The cleaner strips that noise so the residue is something TMDB's
|
||||
// search can match. Heuristic, not perfect.
|
||||
|
||||
/// Cleaned-up title plus extracted hints.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct CleanedTitle {
|
||||
/// Best-effort name suitable for TMDB query.
|
||||
pub(crate) title: String,
|
||||
/// 4-digit year if one was found in the raw title (preserved
|
||||
/// before stripping). Helps TMDB disambiguate remakes.
|
||||
pub(crate) year: Option<u16>,
|
||||
/// `Tv` when the title carries an episode marker (`S01E02`,
|
||||
/// trailing `- 117`, …); `Movie` otherwise. The kind hint flips
|
||||
/// which TMDB endpoint we hit.
|
||||
pub(crate) kind: TmdbKind,
|
||||
}
|
||||
|
||||
/// Junk tokens to drop from torrent titles when cleaning for TMDB
|
||||
/// search. ASCII case-insensitive match against whitespace-split
|
||||
/// tokens AND dot-separated tokens (`a.b.c` → ["a","b","c"]).
|
||||
// Kept in parity with the core torznab plugin; unused in the current
|
||||
// title-cleaning path (the active path uses `filename_meta`).
|
||||
#[allow(dead_code)]
|
||||
pub(crate) const JUNK_TOKENS: &[&str] = &[
|
||||
// resolutions
|
||||
"1080p",
|
||||
"720p",
|
||||
"480p",
|
||||
"2160p",
|
||||
"4k",
|
||||
"uhd",
|
||||
"hd",
|
||||
// sources
|
||||
"bluray",
|
||||
"bdrip",
|
||||
"bdremux",
|
||||
"remux",
|
||||
"web-dl",
|
||||
"webdl",
|
||||
"webrip",
|
||||
"hdtv",
|
||||
"dvdrip",
|
||||
"hdrip",
|
||||
"dsnp",
|
||||
"amzn",
|
||||
"atvp",
|
||||
"nf",
|
||||
"hulu",
|
||||
"adn",
|
||||
// codecs
|
||||
"x264",
|
||||
"x265",
|
||||
"h264",
|
||||
"h265",
|
||||
"hevc",
|
||||
"avc",
|
||||
"10bit",
|
||||
"10-bit",
|
||||
"8bit",
|
||||
"8-bit",
|
||||
// audio
|
||||
"aac",
|
||||
"ac3",
|
||||
"flac",
|
||||
"dts",
|
||||
"dd5",
|
||||
"ddp",
|
||||
"ddp5",
|
||||
"ddp7",
|
||||
"atmos",
|
||||
"mp3",
|
||||
"opus",
|
||||
"5.1",
|
||||
"7.1",
|
||||
"2.0",
|
||||
// subs / language
|
||||
"multi",
|
||||
"multisub",
|
||||
"multi-sub",
|
||||
"multi-subs",
|
||||
"vostfr",
|
||||
"dual-audio",
|
||||
"dualaudio",
|
||||
"sub",
|
||||
"subs",
|
||||
"subbed",
|
||||
// common file extensions
|
||||
"mkv",
|
||||
"mp4",
|
||||
"avi",
|
||||
"ts",
|
||||
"m4v",
|
||||
// misc release tags
|
||||
"proper",
|
||||
"repack",
|
||||
"internal",
|
||||
"limited",
|
||||
"extended",
|
||||
"uncut",
|
||||
"directors",
|
||||
"remastered",
|
||||
"anniversary",
|
||||
"ova",
|
||||
"oad",
|
||||
"ona",
|
||||
"movie",
|
||||
];
|
||||
|
||||
/// Clean a raw torrent title into a TMDB-searchable shape. Pipeline:
|
||||
///
|
||||
/// 1. Strip every `[...]` and `(...)` segment (release groups, tags).
|
||||
/// 2. Strip trailing file extension.
|
||||
/// 3. Detect SxxEyy markers (→ TV) and trailing ` - N` episode numbers (→ TV).
|
||||
/// 4. Extract a year if one is visible.
|
||||
/// 5. Split into tokens (whitespace + dots + hyphens treated as
|
||||
/// separators), drop junk keywords (case-insensitive), drop
|
||||
/// lone-year tokens after extraction, drop empty tokens.
|
||||
/// 6. Re-assemble the survivors with single-space separators.
|
||||
pub(crate) fn clean_torrent_title(raw: &str) -> CleanedTitle {
|
||||
let kind = detect_kind(&strip_file_extension(&strip_bracketed(raw)));
|
||||
// Year from the RAW string — `strip_bracketed` would eat a `(2023)`.
|
||||
let year = extract_year(raw);
|
||||
// Title via the *tested* filename-tools `cleanTitle` pipeline rather than a
|
||||
// second, weaker hand-rolled cleaner. This normalizes `_`/`.` separators,
|
||||
// strips the full release/quality keyword list, drops everything after the
|
||||
// first " - " delimiter, and removes season-word / episode-range markers —
|
||||
// all the cases the old cleaner left in, which made TMDB matching fail.
|
||||
let title = filename_meta::clean_title(raw);
|
||||
CleanedTitle { title, year, kind }
|
||||
}
|
||||
|
||||
pub(crate) fn strip_bracketed(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut depth_sq = 0i32;
|
||||
let mut depth_pa = 0i32;
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'[' => depth_sq += 1,
|
||||
']' => {
|
||||
if depth_sq > 0 {
|
||||
depth_sq -= 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
'(' => depth_pa += 1,
|
||||
')' => {
|
||||
if depth_pa > 0 {
|
||||
depth_pa -= 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
_ if depth_sq > 0 || depth_pa > 0 => continue,
|
||||
_ => {}
|
||||
}
|
||||
if c != '[' && c != '(' {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub(crate) fn strip_file_extension(s: &str) -> String {
|
||||
let trimmed = s.trim();
|
||||
for ext in ["mkv", "mp4", "avi", "ts", "m4v", "webm"] {
|
||||
let suffix = format!(".{ext}");
|
||||
if trimmed.to_ascii_lowercase().ends_with(suffix.as_str()) {
|
||||
return trimmed[..trimmed.len() - suffix.len()].to_string();
|
||||
}
|
||||
}
|
||||
trimmed.to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn detect_kind(s: &str) -> TmdbKind {
|
||||
// Walk tokens (anything not alphanumeric is a separator —
|
||||
// matches dots, spaces, brackets, hyphens). A token of shape
|
||||
// S\d{1,2}, SxxEyy, EpNN, or `Season` triggers TV.
|
||||
let lower = s.to_ascii_lowercase();
|
||||
for tok in lower.split(|c: char| !c.is_ascii_alphanumeric()) {
|
||||
if tok.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if tok == "season" {
|
||||
return TmdbKind::Tv;
|
||||
}
|
||||
// SxxEyy or Sxx (season-only).
|
||||
if let Some(rest) = tok.strip_prefix('s') {
|
||||
let digits_then = rest.chars().take_while(|c| c.is_ascii_digit()).count();
|
||||
if (1..=2).contains(&digits_then) {
|
||||
let after = &rest[digits_then..];
|
||||
if after.is_empty() {
|
||||
return TmdbKind::Tv;
|
||||
}
|
||||
if after.starts_with('e')
|
||||
&& after[1..].chars().all(|c| c.is_ascii_digit())
|
||||
&& !after[1..].is_empty()
|
||||
{
|
||||
return TmdbKind::Tv;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Trailing " - N" / " - NNN" episode-number pattern
|
||||
// (common Nyaa shape: `[Group] Show - 117 [1080p].mkv`).
|
||||
if has_trailing_episode_number(s) {
|
||||
return TmdbKind::Tv;
|
||||
}
|
||||
TmdbKind::Movie
|
||||
}
|
||||
|
||||
pub(crate) fn has_trailing_episode_number(s: &str) -> bool {
|
||||
let trimmed = s.trim();
|
||||
// Look for ` - <digits>` near the end of the string.
|
||||
if let Some(idx) = trimmed.rfind(" - ") {
|
||||
let tail = trimmed[idx + 3..].trim();
|
||||
// Tail is digits only, possibly followed by a `(...)` segment
|
||||
// (already stripped) or `v<digit>` version suffix.
|
||||
let mut digits = 0;
|
||||
for c in tail.chars() {
|
||||
if c.is_ascii_digit() {
|
||||
digits += 1;
|
||||
} else if (c == 'v' || c == 'V') && digits > 0 {
|
||||
break;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return digits > 0 && digits <= 4;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // parity with core torznab; unused in the active title path
|
||||
pub(crate) fn strip_episode_markers(s: &str) -> String {
|
||||
let trimmed = s.trim();
|
||||
// Drop a trailing ` - N` / ` - NN` / ` - NNN` / ` - NNNN[vN]`.
|
||||
if let Some(idx) = trimmed.rfind(" - ") {
|
||||
let tail = &trimmed[idx + 3..];
|
||||
if tail
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_digit() || c == 'v' || c == 'V')
|
||||
&& tail.chars().any(|c| c.is_ascii_digit())
|
||||
{
|
||||
return trimmed[..idx].trim().to_string();
|
||||
}
|
||||
}
|
||||
// Drop SxxEyy and trailing season-only markers.
|
||||
let mut out = String::with_capacity(trimmed.len());
|
||||
let mut chars = trimmed.chars().peekable();
|
||||
while let Some(c) = chars.next() {
|
||||
if (c == 'S' || c == 's') && chars.peek().is_some_and(|n| n.is_ascii_digit()) {
|
||||
// Try to consume Sxx[Eyy].
|
||||
let mut probe = String::new();
|
||||
probe.push(c);
|
||||
while let Some(&n) = chars.peek() {
|
||||
if n.is_ascii_digit() || n == 'E' || n == 'e' {
|
||||
probe.push(n);
|
||||
chars.next();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Validate shape: starts with S, has digits, optional E + digits.
|
||||
let lower = probe.to_ascii_lowercase();
|
||||
let looks_like_marker = lower.starts_with('s')
|
||||
&& lower[1..].chars().take(2).all(|c| c.is_ascii_digit())
|
||||
&& (lower.len() == 3 || lower.contains('e'));
|
||||
if looks_like_marker {
|
||||
continue;
|
||||
}
|
||||
// Not a marker — push back as best we can.
|
||||
out.push_str(&probe);
|
||||
} else {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
pub(crate) fn extract_year(s: &str) -> Option<u16> {
|
||||
// Find a 4-digit standalone token between 1900 and 2099.
|
||||
for tok in s.split(|c: char| !c.is_ascii_alphanumeric()) {
|
||||
if tok.len() == 4 && tok.chars().all(|c| c.is_ascii_digit()) {
|
||||
if let Ok(y) = tok.parse::<u16>() {
|
||||
if (1900..=2099).contains(&y) {
|
||||
return Some(y);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // parity with core torznab; unused in the active title path
|
||||
pub(crate) fn tokenize_and_filter(s: &str, extracted_year: Option<u16>) -> String {
|
||||
let extracted_year_str = extracted_year.map(|y| y.to_string());
|
||||
let mut kept: Vec<String> = Vec::new();
|
||||
for tok in s.split(|c: char| !c.is_ascii_alphanumeric() && c != '\'') {
|
||||
let trimmed = tok.trim_matches(|c: char| !c.is_ascii_alphanumeric() && c != '\'');
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
if JUNK_TOKENS.iter().any(|j| *j == lower) {
|
||||
continue;
|
||||
}
|
||||
// Drop the year token if we extracted it.
|
||||
if let Some(y) = extracted_year_str.as_deref() {
|
||||
if trimmed == y {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Drop pure-digit tokens longer than 2 chars (likely episode
|
||||
// numbers that slipped through, or stray years not matching).
|
||||
if trimmed.len() > 2 && trimmed.chars().all(|c| c.is_ascii_digit()) {
|
||||
continue;
|
||||
}
|
||||
kept.push(trimmed.to_string());
|
||||
}
|
||||
kept.join(" ").trim().to_string()
|
||||
}
|
||||
@@ -0,0 +1,851 @@
|
||||
//! TMDB HTTP client + DTOs + season/episode bounds check.
|
||||
//!
|
||||
//! Split out of the monolithic `torznab.rs` (pure file move; no behaviour change).
|
||||
|
||||
use meta_feeder_sdk::common::urlencode;
|
||||
use std::time::Duration;
|
||||
|
||||
use tracing::debug;
|
||||
|
||||
// -- TMDB metadata enrichment ------------------------------------------------
|
||||
//
|
||||
// TMDB v4 bearer-token client. Two endpoints used:
|
||||
//
|
||||
// GET /3/search/{movie,tv}?query=<urlencoded>&include_adult=false[&year=<YYYY>]
|
||||
// GET <image_base>/<poster_path>
|
||||
//
|
||||
// The token-bearing `Authorization` header gates both. Free-tier rate
|
||||
// limit is ~40 req / 10 s — we don't hit it from a dev box but it's
|
||||
// worth knowing. All TMDB failures degrade silently: enrichment is a
|
||||
// best-effort augmentation, never a hard dep.
|
||||
|
||||
/// Lightweight TMDB v4 client. Cheap to clone (the underlying
|
||||
/// `reqwest::Client` is `Arc`-backed).
|
||||
#[derive(Clone)]
|
||||
pub struct TmdbClient {
|
||||
http: reqwest::Client,
|
||||
bearer_token: String,
|
||||
api_base: String,
|
||||
image_base: String,
|
||||
}
|
||||
|
||||
impl TmdbClient {
|
||||
pub fn new(bearer_token: String) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(TMDB_POSTER_TIMEOUT_SECS))
|
||||
.user_agent(USER_AGENT)
|
||||
.build()
|
||||
.expect("rustls reqwest client build infallible");
|
||||
Self {
|
||||
http,
|
||||
bearer_token,
|
||||
api_base: TMDB_API_BASE.to_string(),
|
||||
image_base: TMDB_IMAGE_BASE.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_bases(token: String, api_base: String, image_base: String) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(TMDB_POSTER_TIMEOUT_SECS))
|
||||
.user_agent(USER_AGENT)
|
||||
.build()
|
||||
.expect("rustls reqwest client build infallible");
|
||||
Self {
|
||||
http,
|
||||
bearer_token: token,
|
||||
api_base,
|
||||
image_base,
|
||||
}
|
||||
}
|
||||
|
||||
/// Search TMDB for a movie/TV title. Returns [`TmdbCall::Hit`] with the
|
||||
/// top result, [`TmdbCall::Miss`] when there are no matches (or any
|
||||
/// transient/timeout error — best-effort degrade), or
|
||||
/// [`TmdbCall::RateLimited`] carrying the Retry-After window on a 429 so
|
||||
/// the caller can pause the shared budget globally. Wraps the whole call
|
||||
/// (including JSON decode) in [`TMDB_SEARCH_TIMEOUT_SECS`].
|
||||
pub(crate) async fn search(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
title: &str,
|
||||
year: Option<u16>,
|
||||
) -> TmdbCall<TmdbHit> {
|
||||
let endpoint = match kind {
|
||||
TmdbKind::Movie => "search/movie",
|
||||
TmdbKind::Tv => "search/tv",
|
||||
};
|
||||
let mut url = format!(
|
||||
"{}/{}?query={}&include_adult=false",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
endpoint,
|
||||
urlencode(title),
|
||||
);
|
||||
if let Some(y) = year {
|
||||
// Movie uses `year`, TV uses `first_air_date_year`; both
|
||||
// are documented but movies accept `year` for backward-
|
||||
// compat which is what we want.
|
||||
url.push_str(&format!("&year={y}"));
|
||||
}
|
||||
let fut = self
|
||||
.http
|
||||
.get(&url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb search non-2xx; degrading"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
let body: TmdbSearchResponse =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json())
|
||||
.await
|
||||
{
|
||||
Ok(Ok(b)) => b,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
match body.results.into_iter().next() {
|
||||
Some(top) => TmdbCall::Hit(top.into_hit(kind)),
|
||||
None => TmdbCall::Miss,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the public TMDB poster CDN URL for `poster_path` (no fetch). In the
|
||||
/// feeder model the feeder no longer fetches+stores the poster bytes; it
|
||||
/// emits this URL as a `poster_url` field and the **gateway core** seeds it
|
||||
/// into a content-addressed `poster` cid (same path as giphy/wikicommons
|
||||
/// previews). Inserts the required [`TMDB_POSTER_SIZE`] segment.
|
||||
pub(crate) fn poster_cdn_url(&self, poster_path: &str) -> String {
|
||||
format!(
|
||||
"{}/{}/{}",
|
||||
self.image_base.trim_end_matches('/'),
|
||||
TMDB_POSTER_SIZE,
|
||||
poster_path.trim_start_matches('/'),
|
||||
)
|
||||
}
|
||||
|
||||
/// Fetch a poster image's raw bytes. `poster_path` is the
|
||||
/// TMDB-supplied path (typically `"/abcdef.jpg"` with leading slash).
|
||||
///
|
||||
/// `image_base` is the bare CDN root (`https://image.tmdb.org/t/p`); TMDB
|
||||
/// requires a **size segment** (`/w500`, `/original`, …) between the root
|
||||
/// and the file or it 404s. The const comment documents this contract and
|
||||
/// this caller honors it by inserting [`TMDB_POSTER_SIZE`].
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) async fn fetch_poster(&self, poster_path: &str) -> Option<bytes::Bytes> {
|
||||
let url = format!(
|
||||
"{}/{}/{}",
|
||||
self.image_base.trim_end_matches('/'),
|
||||
TMDB_POSTER_SIZE,
|
||||
poster_path.trim_start_matches('/'),
|
||||
);
|
||||
let resp = self.http.get(&url).send().await.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
poster_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb poster fetch non-2xx; skipping"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
resp.bytes().await.ok()
|
||||
}
|
||||
|
||||
/// Fetch a TV show's authoritative structure (`number_of_seasons`
|
||||
/// plus the per-season `episode_count`s) via `GET /3/tv/{id}`. Used
|
||||
/// only to bounds-check a title-parsed season/episode — see
|
||||
/// [`season_episode_bounds`]. Wrapped in the same search timeout;
|
||||
/// any failure returns `None` so validation degrades to "accept".
|
||||
pub(crate) async fn tv_details(&self, tmdbid: u64) -> TmdbCall<TmdbTvDetails> {
|
||||
let url = format!("{}/tv/{}", self.api_base.trim_end_matches('/'), tmdbid,);
|
||||
let fut = self
|
||||
.http
|
||||
.get(&url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
_ => return TmdbCall::Miss,
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
"tmdb tv-details non-2xx; skipping season/episode validation"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json()).await
|
||||
{
|
||||
Ok(Ok(d)) => TmdbCall::Hit(d),
|
||||
_ => TmdbCall::Miss,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch cross-database ids (`tvdb_id`, `imdb_id`) for a known tmdbid via
|
||||
/// `GET /3/{tv,movie}/{id}/external_ids`. The anchored torznab path needs
|
||||
/// `tvdb_id` to query indexers by `tvsearch&tvdbid=` (TMDB's id is not a
|
||||
/// standard `tvsearch` param). Same timeout/429 contract as [`tv_details`].
|
||||
pub(crate) async fn external_ids(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
tmdbid: u64,
|
||||
) -> TmdbCall<TmdbExternalIds> {
|
||||
let seg = match kind {
|
||||
TmdbKind::Movie => "movie",
|
||||
TmdbKind::Tv => "tv",
|
||||
};
|
||||
let url = format!(
|
||||
"{}/{}/{}/external_ids",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
seg,
|
||||
tmdbid,
|
||||
);
|
||||
self.get_json(&url, "external_ids").await
|
||||
}
|
||||
|
||||
/// Fetch a movie's canonical details via `GET /3/movie/{id}` (the anchored
|
||||
/// enrichment source for a known movie tmdbid — resolves title/overview/
|
||||
/// poster/year/imdb_id without a fuzzy title search). Same contract as
|
||||
/// [`tv_details`].
|
||||
pub(crate) async fn movie_details(&self, tmdbid: u64) -> TmdbCall<TmdbMovieDetails> {
|
||||
let url = format!("{}/movie/{}", self.api_base.trim_end_matches('/'), tmdbid);
|
||||
self.get_json(&url, "movie-details").await
|
||||
}
|
||||
|
||||
/// Principal search via `GET /3/search/multi` — maps a bare keyword to a
|
||||
/// canonical tmdbid + media type so the gateway can then query indexers
|
||||
/// structurally (the "TMDB as the front door" identification step
|
||||
/// Sonarr/Radarr use). Returns the mixed movie/tv/person result list; the
|
||||
/// caller picks the top-N confident anchors via [`principal_top_n`].
|
||||
pub(crate) async fn search_multi(&self, query: &str) -> TmdbCall<Vec<TmdbMultiItem>> {
|
||||
let url = format!(
|
||||
"{}/search/multi?query={}&include_adult=false",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
urlencode(query),
|
||||
);
|
||||
match self
|
||||
.get_json::<TmdbMultiResponse>(&url, "search/multi")
|
||||
.await
|
||||
{
|
||||
TmdbCall::Hit(r) => TmdbCall::Hit(r.results),
|
||||
TmdbCall::Miss => TmdbCall::Miss,
|
||||
TmdbCall::RateLimited(d) => TmdbCall::RateLimited(d),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a TMDB **discovery list** (trending / popular / top-rated / a
|
||||
/// `/discover` query) and map each result to a [`TmdbHit`]. Powers the
|
||||
/// keyword-less discovery branch (see [`super::discovery`]): the caller
|
||||
/// composes `path_and_query` (e.g. `"movie/popular"`,
|
||||
/// `"trending/tv/week"`, or `"discover/tv?with_keywords=210024&sort_by=…"`)
|
||||
/// and passes the `kind` so movie-vs-TV title/date fields decode correctly.
|
||||
/// All these endpoints share the `{results: [TmdbSearchItem]}` shape, so one
|
||||
/// method covers them. Returns the page's hits, or `Miss`/`RateLimited` under
|
||||
/// the usual best-effort contract.
|
||||
pub(crate) async fn discovery_list(
|
||||
&self,
|
||||
kind: TmdbKind,
|
||||
path_and_query: &str,
|
||||
) -> TmdbCall<Vec<TmdbHit>> {
|
||||
let url = format!(
|
||||
"{}/{}",
|
||||
self.api_base.trim_end_matches('/'),
|
||||
path_and_query.trim_start_matches('/'),
|
||||
);
|
||||
match self.get_json::<TmdbSearchResponse>(&url, "discovery").await {
|
||||
TmdbCall::Hit(r) => {
|
||||
TmdbCall::Hit(r.results.into_iter().map(|i| i.into_hit(kind)).collect())
|
||||
}
|
||||
TmdbCall::Miss => TmdbCall::Miss,
|
||||
TmdbCall::RateLimited(d) => TmdbCall::RateLimited(d),
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared `GET <url>` → JSON helper with the standard bearer auth, search
|
||||
/// timeout, 429→`RateLimited`, non-2xx/transient→`Miss` contract. Factored
|
||||
/// out of [`tv_details`]/[`external_ids`]/[`movie_details`] (identical
|
||||
/// modulo the decoded type and the log label).
|
||||
async fn get_json<T: serde::de::DeserializeOwned>(&self, url: &str, what: &str) -> TmdbCall<T> {
|
||||
let fut = self
|
||||
.http
|
||||
.get(url)
|
||||
.bearer_auth(&self.bearer_token)
|
||||
.header("accept", "application/json")
|
||||
.send();
|
||||
let resp =
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), fut).await {
|
||||
Ok(Ok(r)) => r,
|
||||
// Transport error / timeout. Logged at debug so the otherwise
|
||||
// silent best-effort degrade is diagnosable.
|
||||
Ok(Err(e)) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, error=%e, "tmdb call send error; degrading");
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, "tmdb call send timeout; degrading");
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
};
|
||||
if resp.status().as_u16() == 429 {
|
||||
return TmdbCall::RateLimited(parse_tmdb_retry_after(&resp));
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
debug!(
|
||||
target: "meta-share::gateway",
|
||||
upstream = "prowlarr",
|
||||
tmdb_url = %url,
|
||||
status = %resp.status(),
|
||||
what,
|
||||
"tmdb call non-2xx; degrading"
|
||||
);
|
||||
return TmdbCall::Miss;
|
||||
}
|
||||
match tokio::time::timeout(Duration::from_secs(TMDB_SEARCH_TIMEOUT_SECS), resp.json()).await {
|
||||
Ok(Ok(d)) => TmdbCall::Hit(d),
|
||||
// A decode failure here folds into `Miss` (best-effort), but log it
|
||||
// at debug: a single malformed element fails the whole `Vec` decode,
|
||||
// which would otherwise silently disable e.g. multi-anchor.
|
||||
Ok(Err(e)) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, error=%e, "tmdb call decode error; degrading");
|
||||
TmdbCall::Miss
|
||||
}
|
||||
Err(_) => {
|
||||
debug!(target: "meta-share::gateway", upstream="prowlarr", tmdb_url=%url, what, "tmdb call decode timeout; degrading");
|
||||
TmdbCall::Miss
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of a TMDB API call. Distinguishes a 429 rate-limit (which feeds
|
||||
/// `Retry-After` into the shared [`TmdbBudget`] so all enrichment pauses
|
||||
/// globally) from a plain miss or transient error (best-effort degrade).
|
||||
pub(crate) enum TmdbCall<T> {
|
||||
Hit(T),
|
||||
Miss,
|
||||
RateLimited(Duration),
|
||||
}
|
||||
|
||||
/// Parse a TMDB `Retry-After` header into a pause duration, defaulting to
|
||||
/// [`TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS`] when absent/unparseable. Mirrors
|
||||
/// the indexer-path logic in [`map_status`].
|
||||
pub(crate) fn parse_tmdb_retry_after(resp: &reqwest::Response) -> Duration {
|
||||
let secs = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS);
|
||||
Duration::from_secs(secs)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum TmdbKind {
|
||||
Movie,
|
||||
Tv,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbSearchResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) results: Vec<TmdbSearchItem>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbSearchItem {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) original_name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) release_date: Option<String>, // movie, YYYY-MM-DD
|
||||
#[serde(default)]
|
||||
pub(crate) first_air_date: Option<String>, // tv, YYYY-MM-DD
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) genre_ids: Vec<u32>,
|
||||
/// ISO 639-1 (2-letter) language the title was originally produced in.
|
||||
/// Mapped to `lang3` and used to file `original_title` under
|
||||
/// `titles/{lang3}` (METADATA_KEYS.md §3).
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
}
|
||||
|
||||
impl TmdbSearchItem {
|
||||
pub(crate) fn into_hit(self, kind: TmdbKind) -> TmdbHit {
|
||||
let title = match kind {
|
||||
TmdbKind::Movie => self.title,
|
||||
TmdbKind::Tv => self.name,
|
||||
}
|
||||
.unwrap_or_else(|| "(untitled)".to_string());
|
||||
let original_title = match kind {
|
||||
TmdbKind::Movie => self.original_title,
|
||||
TmdbKind::Tv => self.original_name,
|
||||
};
|
||||
let date = match kind {
|
||||
TmdbKind::Movie => self.release_date,
|
||||
TmdbKind::Tv => self.first_air_date,
|
||||
};
|
||||
let year = date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
TmdbHit {
|
||||
tmdbid: self.id,
|
||||
title,
|
||||
original_title,
|
||||
original_language: self.original_language,
|
||||
overview: self.overview,
|
||||
year,
|
||||
poster_path: self.poster_path,
|
||||
genre_ids: self.genre_ids,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Map an ISO 639-1 (2-letter) code to its ISO 639-3 (`lang3`) equivalent,
|
||||
/// matching the store's convention (`eng`, `jpn`, `fra`; the 639-2/T variant
|
||||
/// where B/T differ, e.g. `deu` not `ger`). Covers the languages TMDB
|
||||
/// commonly returns; unknown codes return `None` so the caller skips the
|
||||
/// `titles/{lang3}` write rather than persisting a non-`lang3` key.
|
||||
pub(crate) fn iso639_1_to_3(code: &str) -> Option<&'static str> {
|
||||
Some(match code.to_ascii_lowercase().as_str() {
|
||||
"en" => "eng",
|
||||
"ja" => "jpn",
|
||||
"fr" => "fra",
|
||||
"de" => "deu",
|
||||
"es" => "spa",
|
||||
"it" => "ita",
|
||||
"ru" => "rus",
|
||||
"ko" => "kor",
|
||||
"zh" => "zho",
|
||||
"pt" => "por",
|
||||
"nl" => "nld",
|
||||
"sv" => "swe",
|
||||
"no" => "nor",
|
||||
"da" => "dan",
|
||||
"fi" => "fin",
|
||||
"pl" => "pol",
|
||||
"tr" => "tur",
|
||||
"ar" => "ara",
|
||||
"hi" => "hin",
|
||||
"th" => "tha",
|
||||
"vi" => "vie",
|
||||
"id" => "ind",
|
||||
"cs" => "ces",
|
||||
"el" => "ell",
|
||||
"he" => "heb",
|
||||
"hu" => "hun",
|
||||
"ro" => "ron",
|
||||
"uk" => "ukr",
|
||||
"fa" => "fas",
|
||||
"ms" => "msa",
|
||||
"tl" => "tgl",
|
||||
"ca" => "cat",
|
||||
"nb" => "nob",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
// `Clone` is required so the single-flight `Shared` future (whose `Output`
|
||||
// must be `Clone`) can hand the same hit to every coalesced caller; serde so
|
||||
// hits persist in the redb TMDB-search cache.
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbHit {
|
||||
pub(crate) tmdbid: u64,
|
||||
pub(crate) title: String,
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) year: Option<u16>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
#[serde(default)]
|
||||
#[allow(dead_code)] // surfaced as CSV of ids; name resolution would need a second call
|
||||
pub(crate) genre_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
impl TmdbHit {
|
||||
/// TMDB `original_language` (ISO 639-1) mapped to the store's `lang3`
|
||||
/// (ISO 639-3), or `None` when the language is outside the common set or
|
||||
/// absent. Used to file `original_title` under `titles/{lang3}` (§3).
|
||||
pub(crate) fn original_lang3(&self) -> Option<&'static str> {
|
||||
iso639_1_to_3(self.original_language.as_deref()?.trim())
|
||||
}
|
||||
}
|
||||
|
||||
/// Authoritative TV structure from `GET /3/tv/{id}`, used to bounds-check
|
||||
/// a title-parsed season/episode. Only the two structural fields are
|
||||
/// decoded; everything else in the (large) details payload is ignored.
|
||||
// `Clone` + `Serialize` added so details persist in the redb TMDB-tvdetails
|
||||
// cache and can be cheaply handed around.
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbTvDetails {
|
||||
#[serde(default)]
|
||||
pub(crate) number_of_seasons: u32,
|
||||
/// One entry per season TMDB knows about, including season 0
|
||||
/// ("Specials"). `episode_count` is the released-episode total.
|
||||
#[serde(default)]
|
||||
pub(crate) seasons: Vec<TmdbSeasonSummary>,
|
||||
// -- Display fields ------------------------------------------------------
|
||||
// Decoded from the same `GET /3/tv/{id}` payload so an anchored TV record
|
||||
// (known tmdbid) enriches directly from its canonical entry instead of a
|
||||
// fuzzy title search. Old cache entries (written before these fields
|
||||
// existed) deserialize them to empty/None; [`TmdbTvDetails::has_display`]
|
||||
// detects that so the cached-fetch path can self-heal with one refetch.
|
||||
#[serde(default)]
|
||||
pub(crate) name: String,
|
||||
#[serde(default)]
|
||||
pub(crate) original_name: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) first_air_date: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
}
|
||||
|
||||
impl TmdbTvDetails {
|
||||
/// True when the cached payload carries the display fields (i.e. was
|
||||
/// written by the current decoder). A `false` here on a cache hit means a
|
||||
/// pre-display entry — the caller refetches once to upgrade it.
|
||||
pub(crate) fn has_display(&self) -> bool {
|
||||
!self.name.trim().is_empty()
|
||||
}
|
||||
|
||||
/// Build a [`TmdbHit`] (resolved by id) from the canonical TV entry, for
|
||||
/// the anchored enrichment path. `None` when display fields are absent.
|
||||
pub(crate) fn as_hit(&self, tmdbid: u64) -> Option<TmdbHit> {
|
||||
if !self.has_display() {
|
||||
return None;
|
||||
}
|
||||
let year = self
|
||||
.first_air_date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
Some(TmdbHit {
|
||||
tmdbid,
|
||||
title: self.name.clone(),
|
||||
original_title: self.original_name.clone(),
|
||||
original_language: self.original_language.clone(),
|
||||
overview: self.overview.clone(),
|
||||
year,
|
||||
poster_path: self.poster_path.clone(),
|
||||
genre_ids: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// TMDB cross-database ids from `GET /3/{tv,movie}/{id}/external_ids`. Only the
|
||||
/// two the anchored torznab path needs are decoded.
|
||||
#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbExternalIds {
|
||||
#[serde(default)]
|
||||
pub(crate) tvdb_id: Option<i64>,
|
||||
#[serde(default)]
|
||||
pub(crate) imdb_id: Option<String>,
|
||||
}
|
||||
|
||||
/// Authoritative movie structure from `GET /3/movie/{id}`. Decodes only the
|
||||
/// display fields the anchored path needs; `imdb_id` (movie details carry it
|
||||
/// inline, no separate `external_ids` call) feeds the `t=movie&imdbid=` query
|
||||
/// fallback.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct TmdbMovieDetails {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_title: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) original_language: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) overview: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) release_date: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) poster_path: Option<String>,
|
||||
}
|
||||
|
||||
impl TmdbMovieDetails {
|
||||
pub(crate) fn into_hit(self) -> TmdbHit {
|
||||
let year = self
|
||||
.release_date
|
||||
.as_deref()
|
||||
.filter(|d| d.len() >= 4)
|
||||
.and_then(|d| d[..4].parse::<u16>().ok());
|
||||
TmdbHit {
|
||||
tmdbid: self.id,
|
||||
title: self.title.unwrap_or_else(|| "(untitled)".to_string()),
|
||||
original_title: self.original_title,
|
||||
original_language: self.original_language,
|
||||
overview: self.overview,
|
||||
year,
|
||||
poster_path: self.poster_path,
|
||||
genre_ids: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
pub(crate) struct TmdbMultiResponse {
|
||||
#[serde(default)]
|
||||
pub(crate) results: Vec<TmdbMultiItem>,
|
||||
}
|
||||
|
||||
/// One `search/multi` result. `media_type` discriminates movie / tv / person;
|
||||
/// we anchor only on movie/tv.
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub(crate) struct TmdbMultiItem {
|
||||
pub(crate) id: u64,
|
||||
#[serde(default)]
|
||||
pub(crate) media_type: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) title: Option<String>, // movie
|
||||
#[serde(default)]
|
||||
pub(crate) name: Option<String>, // tv
|
||||
#[serde(default)]
|
||||
pub(crate) popularity: f64,
|
||||
}
|
||||
|
||||
impl TmdbMultiItem {
|
||||
pub(crate) fn kind(&self) -> Option<TmdbKind> {
|
||||
match self.media_type.as_deref() {
|
||||
Some("movie") => Some(TmdbKind::Movie),
|
||||
Some("tv") => Some(TmdbKind::Tv),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
pub(crate) fn display_title(&self) -> &str {
|
||||
self.title.as_deref().or(self.name.as_deref()).unwrap_or("")
|
||||
}
|
||||
}
|
||||
|
||||
/// Lowercase alphanumeric word tokens of `s` (separators dropped). Shared by
|
||||
/// the principal-search confidence check.
|
||||
pub(crate) fn norm_tokens(s: &str) -> Vec<String> {
|
||||
s.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(|t| t.to_lowercase())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Pick up to `n` **confident** anchors from `search/multi` hits for `query`,
|
||||
/// most-popular first (or `[]` → the caller falls back to today's generic
|
||||
/// `t=search&q=`). For each movie/tv hit, accept it only if its title shares at
|
||||
/// least half of the query's word tokens (the per-candidate relevance gate),
|
||||
/// then rank the survivors by popularity and take the top `n`. This binds a
|
||||
/// vague keyword like "black" to the *several* real shows TMDB returns (Black
|
||||
/// Mirror, Black Butler, Black Lagoon, …) instead of hijacking the whole query
|
||||
/// onto the single most-popular title, while still dropping unrelated noise
|
||||
/// (e.g. "zzz qqq www" → nothing). `n == 1` recovers the old single-anchor
|
||||
/// "principal_confident" behaviour (modulo the gate being applied before the
|
||||
/// popularity pick rather than after).
|
||||
pub(crate) fn principal_top_n<'a>(
|
||||
hits: &'a [TmdbMultiItem],
|
||||
query: &str,
|
||||
n: usize,
|
||||
) -> Vec<&'a TmdbMultiItem> {
|
||||
let q_tokens = norm_tokens(query);
|
||||
if q_tokens.is_empty() || n == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
let mut cands: Vec<&TmdbMultiItem> = hits
|
||||
.iter()
|
||||
.filter(|h| h.kind().is_some())
|
||||
.filter(|h| {
|
||||
let t_tokens = norm_tokens(h.display_title());
|
||||
let overlap = q_tokens.iter().filter(|t| t_tokens.contains(t)).count();
|
||||
(overlap as f64) >= (q_tokens.len() as f64) * 0.5
|
||||
})
|
||||
.collect();
|
||||
cands.sort_by(|a, b| {
|
||||
b.popularity
|
||||
.partial_cmp(&a.popularity)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
cands.truncate(n);
|
||||
cands
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub(crate) struct TmdbSeasonSummary {
|
||||
#[serde(default)]
|
||||
pub(crate) season_number: i64,
|
||||
#[serde(default)]
|
||||
pub(crate) episode_count: u32,
|
||||
}
|
||||
|
||||
/// Verdict of [`season_episode_bounds`] — how a title-parsed `(season,
|
||||
/// episode)` lines up against TMDB's actual structure.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum SeasonEpisodeBounds {
|
||||
/// Consistent with TMDB, or unverifiable — keep the record as-is.
|
||||
Ok,
|
||||
/// The parsed season sits past TMDB's known seasons and isn't otherwise
|
||||
/// listed in `seasons[]`. The canonical case is anime where fansubbers
|
||||
/// number a later cour `S2` of a show TMDB models as a **single** season
|
||||
/// (Frieren, Jujutsu Kaisen, …). The TMDB match itself is sound — only the
|
||||
/// season number is a naming-convention artifact — so the caller keeps the
|
||||
/// record but strips the misleading `season` (the frontend buckets
|
||||
/// season-less episodes into a flat "Episodes" list).
|
||||
SeasonOverflow,
|
||||
/// A *positive* contradiction the season-overflow allowance can't explain:
|
||||
/// a negative season, or an episode beyond a **matched** season's
|
||||
/// `episode_count`. The title parse is wrong — the caller drops the record.
|
||||
Contradiction,
|
||||
}
|
||||
|
||||
/// Pure bounds check: is a title-parsed `(season, episode)` consistent
|
||||
/// with what TMDB says the show actually has? Deliberately lenient —
|
||||
/// it only returns a verdict other than [`SeasonEpisodeBounds::Ok`] on a
|
||||
/// *positive* divergence, so anything ambiguous or unverifiable is accepted:
|
||||
///
|
||||
/// - No parsed season → [`Ok`]. Episode-only titles (absolute-numbered
|
||||
/// anime, trailing `- 117`) are legitimately unbounded; the season-
|
||||
/// based bound doesn't apply.
|
||||
/// - TMDB gave us no structure (`number_of_seasons == 0` and no
|
||||
/// `seasons[]`) → [`Ok`]; we can't contradict what we don't know.
|
||||
/// - Season exceeds `number_of_seasons` AND isn't otherwise listed in
|
||||
/// `seasons[]` → [`SeasonOverflow`] (the `S2`-of-a-1-TMDB-season anime
|
||||
/// case; keep but strip `season`).
|
||||
/// - Negative season → [`Contradiction`].
|
||||
/// - Episode exceeds the matched season's `episode_count` (when that
|
||||
/// count is known) → [`Contradiction`]. This only fires for explicit
|
||||
/// `SxxEyy` titles, the exact case where the per-season bound holds.
|
||||
///
|
||||
/// [`Ok`]: SeasonEpisodeBounds::Ok
|
||||
/// [`SeasonOverflow`]: SeasonEpisodeBounds::SeasonOverflow
|
||||
/// [`Contradiction`]: SeasonEpisodeBounds::Contradiction
|
||||
pub(crate) fn season_episode_bounds(
|
||||
details: &TmdbTvDetails,
|
||||
season: Option<i64>,
|
||||
episode: Option<i64>,
|
||||
) -> SeasonEpisodeBounds {
|
||||
use SeasonEpisodeBounds::*;
|
||||
let season = match season {
|
||||
Some(s) => s,
|
||||
None => return Ok,
|
||||
};
|
||||
if details.number_of_seasons == 0 && details.seasons.is_empty() {
|
||||
return Ok;
|
||||
}
|
||||
if season < 0 {
|
||||
return Contradiction;
|
||||
}
|
||||
let matched = details.seasons.iter().find(|s| s.season_number == season);
|
||||
if season > details.number_of_seasons as i64 && matched.is_none() {
|
||||
return SeasonOverflow;
|
||||
}
|
||||
if let (Some(ep), Some(sm)) = (episode, matched) {
|
||||
if sm.episode_count > 0 && ep > sm.episode_count as i64 {
|
||||
return Contradiction;
|
||||
}
|
||||
}
|
||||
Ok
|
||||
}
|
||||
|
||||
/// True when `episode` is a valid episode of `season` per TMDB's per-season
|
||||
/// `episode_count` (the season must be listed and the count known). Used to
|
||||
/// decide whether a bare title number is an in-season episode before falling
|
||||
/// back to absolute-number interpretation.
|
||||
pub(crate) fn episode_in_season(details: &TmdbTvDetails, season: i64, episode: i64) -> bool {
|
||||
details
|
||||
.seasons
|
||||
.iter()
|
||||
.find(|s| s.season_number == season)
|
||||
.map(|s| s.episode_count > 0 && episode >= 1 && episode <= s.episode_count as i64)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
use crate::consts::*;
|
||||
|
||||
#[cfg(test)]
|
||||
mod season_bounds_tests {
|
||||
use super::*;
|
||||
|
||||
fn details(num_seasons: u32, seasons: &[(i64, u32)]) -> TmdbTvDetails {
|
||||
TmdbTvDetails {
|
||||
number_of_seasons: num_seasons,
|
||||
seasons: seasons
|
||||
.iter()
|
||||
.map(|&(season_number, episode_count)| TmdbSeasonSummary {
|
||||
season_number,
|
||||
episode_count,
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_ok_within_season() {
|
||||
let d = details(2, &[(1, 26), (2, 24)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(1), Some(5)), SeasonEpisodeBounds::Ok);
|
||||
assert_eq!(season_episode_bounds(&d, Some(2), Some(24)), SeasonEpisodeBounds::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_no_season_or_unknown_structure_is_ok() {
|
||||
let d = details(2, &[(1, 26), (2, 24)]);
|
||||
assert_eq!(season_episode_bounds(&d, None, Some(999)), SeasonEpisodeBounds::Ok);
|
||||
let unknown = details(0, &[]);
|
||||
assert_eq!(season_episode_bounds(&unknown, Some(5), Some(40)), SeasonEpisodeBounds::Ok);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_season_past_total_is_overflow() {
|
||||
// Fansub "S2" of a show TMDB models as a single season → strip-and-keep.
|
||||
let d = details(1, &[(1, 28)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(2), Some(3)), SeasonEpisodeBounds::SeasonOverflow);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_episode_past_matched_season_or_negative_is_contradiction() {
|
||||
// "S1E27" on a 4-season show → contradiction. The enrich path drops the
|
||||
// record (we no longer remap absolute numbers onto seasons; a genuine
|
||||
// absolute release carries no season token and never reaches here).
|
||||
let d = details(4, &[(1, 26), (2, 13), (3, 13), (4, 12)]);
|
||||
assert_eq!(season_episode_bounds(&d, Some(1), Some(27)), SeasonEpisodeBounds::Contradiction);
|
||||
assert_eq!(season_episode_bounds(&d, Some(-1), None), SeasonEpisodeBounds::Contradiction);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
//! Process-global TMDB request budget.
|
||||
//!
|
||||
//! TMDB's free tier caps at ~40 requests / 10 s. The gateway can field many
|
||||
//! concurrent searches, each wanting to enrich several records, so without a
|
||||
//! shared throttle a burst of discover queries collectively blows the limit
|
||||
//! and TMDB starts returning 429s — which surfaces as empty home rows in
|
||||
//! meta-watch. This is a single hand-rolled token bucket shared by every
|
||||
//! enrichment task in the process (one `Arc<TmdbBudget>` injected into the
|
||||
//! torznab plugin via [`meta_feeder_sdk::plugin::FeederPlugin::set_tmdb_budget`]).
|
||||
//!
|
||||
//! Design notes:
|
||||
//! - **Lazy refill.** Tokens accrue on each `acquire` from elapsed wall-clock,
|
||||
//! so there is no background timer task and no idle cost.
|
||||
//! - **Cancel-safe.** The token is decremented synchronously under the lock
|
||||
//! right before `acquire` returns [`Lease::Granted`]; there is no `.await`
|
||||
//! between the decision to grant and the return, so a dropped `acquire`
|
||||
//! future (consumer disconnected) never spends a token.
|
||||
//! - **Global 429 pause.** A single upstream 429 freezes *all* grants until
|
||||
//! the Retry-After window elapses via [`TmdbBudget::note_429`].
|
||||
//!
|
||||
//! The critical sections hold a plain `std::sync::Mutex` (no awaits inside),
|
||||
//! while the waiting happens outside the lock on a `tokio::time::sleep` raced
|
||||
//! against a `tokio::sync::Notify`.
|
||||
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
/// Default sustained TMDB grant rate (requests/second). Held well under
|
||||
/// TMDB's free-tier ~40 req / 10 s so concurrent searches plus poster-CDN
|
||||
/// headroom never collectively trip a 429.
|
||||
pub const DEFAULT_TMDB_REFILL_PER_SEC: f64 = 3.0;
|
||||
/// Default TMDB burst ceiling (tokens) — a short burst of concurrent searches
|
||||
/// is served immediately, then throttled to the sustained rate.
|
||||
pub const DEFAULT_TMDB_BURST: f64 = 6.0;
|
||||
|
||||
/// Outcome of an [`TmdbBudget::acquire`] call.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Lease {
|
||||
/// A token was granted; the caller may make one TMDB API call.
|
||||
Granted,
|
||||
/// The wait deadline elapsed before a token became available. The caller
|
||||
/// should proceed without enrichment (best-effort degrade).
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
struct BucketState {
|
||||
tokens: f64,
|
||||
capacity: f64,
|
||||
refill_per_sec: f64,
|
||||
last_refill: Instant,
|
||||
/// When set and in the future, all grants are frozen (global 429 pause).
|
||||
paused_until: Option<Instant>,
|
||||
}
|
||||
|
||||
impl BucketState {
|
||||
/// Accrue tokens for elapsed time since the last refill. Called under the
|
||||
/// lock before inspecting `tokens`.
|
||||
fn refill(&mut self, now: Instant) {
|
||||
let elapsed = now
|
||||
.saturating_duration_since(self.last_refill)
|
||||
.as_secs_f64();
|
||||
if elapsed > 0.0 {
|
||||
self.tokens = (self.tokens + elapsed * self.refill_per_sec).min(self.capacity);
|
||||
self.last_refill = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared token bucket gating TMDB API calls across the whole process.
|
||||
pub struct TmdbBudget {
|
||||
state: Mutex<BucketState>,
|
||||
/// Woken whenever a grant *might* now be possible (a token was returned to
|
||||
/// the bucket, or a 429 pause was cleared). Waiters re-check under the
|
||||
/// lock after waking; a missed wake only costs an extra bounded sleep.
|
||||
wake: Notify,
|
||||
}
|
||||
|
||||
impl TmdbBudget {
|
||||
/// Build a budget that sustains `refill_per_sec` grants/second with a burst
|
||||
/// ceiling of `capacity` tokens. The bucket starts full so the first burst
|
||||
/// is served immediately.
|
||||
pub fn new(refill_per_sec: f64, capacity: f64) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(BucketState {
|
||||
tokens: capacity,
|
||||
capacity,
|
||||
refill_per_sec,
|
||||
last_refill: Instant::now(),
|
||||
paused_until: None,
|
||||
}),
|
||||
wake: Notify::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Wait for one token, giving up after `deadline`. Cancel-safe: dropping
|
||||
/// the returned future before it resolves [`Lease::Granted`] spends no
|
||||
/// token.
|
||||
pub async fn acquire(&self, deadline: Duration) -> Lease {
|
||||
let give_up_at = Instant::now() + deadline;
|
||||
loop {
|
||||
// Decide-or-compute-wait, all under the lock (no awaits inside).
|
||||
let sleep_for = {
|
||||
let mut st = self.state.lock().expect("tmdb budget mutex poisoned");
|
||||
let now = Instant::now();
|
||||
if let Some(until) = st.paused_until {
|
||||
if now >= until {
|
||||
st.paused_until = None;
|
||||
}
|
||||
}
|
||||
let paused = st.paused_until;
|
||||
if paused.is_none() {
|
||||
st.refill(now);
|
||||
if st.tokens >= 1.0 {
|
||||
// Atomic grant: decrement and return with no await between.
|
||||
st.tokens -= 1.0;
|
||||
return Lease::Granted;
|
||||
}
|
||||
}
|
||||
if now >= give_up_at {
|
||||
return Lease::DeadlineExceeded;
|
||||
}
|
||||
let until_deadline = give_up_at.saturating_duration_since(now);
|
||||
let wait = match paused {
|
||||
// Frozen: wake when the pause is scheduled to lift.
|
||||
Some(until) => until.saturating_duration_since(now),
|
||||
// Throttled: wake when the next whole token should accrue.
|
||||
None => {
|
||||
let needed = 1.0 - st.tokens;
|
||||
let secs = if st.refill_per_sec > 0.0 {
|
||||
needed / st.refill_per_sec
|
||||
} else {
|
||||
until_deadline.as_secs_f64()
|
||||
};
|
||||
Duration::from_secs_f64(secs.max(0.0))
|
||||
}
|
||||
};
|
||||
wait.min(until_deadline)
|
||||
};
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(sleep_for) => {}
|
||||
_ = self.wake.notified() => {}
|
||||
}
|
||||
if Instant::now() >= give_up_at {
|
||||
return Lease::DeadlineExceeded;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Feed back an upstream 429: pause all grants until `now + retry_after`.
|
||||
/// Extends an existing pause but never shortens it.
|
||||
pub fn note_429(&self, retry_after: Duration) {
|
||||
let until = Instant::now() + retry_after;
|
||||
{
|
||||
let mut st = self.state.lock().expect("tmdb budget mutex poisoned");
|
||||
st.paused_until = Some(match st.paused_until {
|
||||
Some(existing) if existing > until => existing,
|
||||
_ => until,
|
||||
});
|
||||
}
|
||||
// Wake waiters so they recompute their sleep against the new pause.
|
||||
self.wake.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn grants_up_to_capacity_immediately() {
|
||||
let b = TmdbBudget::new(3.0, 3.0);
|
||||
// Three tokens available at t0 — all granted without waiting.
|
||||
for _ in 0..3 {
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn fourth_token_waits_for_refill() {
|
||||
let b = TmdbBudget::new(2.0, 2.0); // 2/s sustained
|
||||
for _ in 0..2 {
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
}
|
||||
// Bucket empty; next grant must wait ~0.5s for one token at 2/s.
|
||||
let start = Instant::now();
|
||||
assert_eq!(b.acquire(Duration::from_secs(10)).await, Lease::Granted);
|
||||
let waited = start.elapsed();
|
||||
assert!(
|
||||
waited >= Duration::from_millis(450),
|
||||
"expected to wait ~0.5s for refill, waited {waited:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn deadline_exceeded_when_drained() {
|
||||
let b = TmdbBudget::new(0.001, 1.0); // effectively no refill within the test
|
||||
assert_eq!(b.acquire(Duration::from_secs(1)).await, Lease::Granted);
|
||||
// Bucket empty, refill far slower than the deadline.
|
||||
assert_eq!(
|
||||
b.acquire(Duration::from_millis(200)).await,
|
||||
Lease::DeadlineExceeded
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn note_429_pauses_grants_globally() {
|
||||
let b = TmdbBudget::new(100.0, 100.0); // plenty of tokens
|
||||
b.note_429(Duration::from_secs(2));
|
||||
// Even with tokens available, grants are frozen during the pause —
|
||||
// a short deadline times out.
|
||||
assert_eq!(
|
||||
b.acquire(Duration::from_millis(500)).await,
|
||||
Lease::DeadlineExceeded
|
||||
);
|
||||
// After the pause window, grants resume.
|
||||
assert_eq!(b.acquire(Duration::from_secs(5)).await, Lease::Granted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Tribler metainfo (file-list) fetch + parsing for the metainfo fan-out
|
||||
//! (PR2 step 3). Turns one torrent infohash into the list of files it contains,
|
||||
//! so `compute_outcomes` can emit one `btih-v1-file` CID per video file plus a
|
||||
//! whole-torrent pack CID.
|
||||
//!
|
||||
//! Source: Tribler core `POST /api/torrentinfo/uri` with a JSON body
|
||||
//! `{"uri": "<magnet>"}` (the `uri` query param alone 500s — the handler reads
|
||||
//! the body as JSON). Response shape (`GetMetainfoResponse`):
|
||||
//!
|
||||
//! ```json
|
||||
//! { "files": [ {"index": 0, "name": "Show.S01E01.mkv", "size": 123}, … ],
|
||||
//! "name": "Show Complete Season 1", "description": "", … }
|
||||
//! ```
|
||||
//!
|
||||
//! The DHT metainfo resolve is **slow and may fail** (no peers / no metadata);
|
||||
//! callers treat `None` as "couldn't enumerate, fall back to the single-file
|
||||
//! record" — a fan-out failure must never fail the resolve.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One file inside a torrent. `index` is the zero-based position in the info
|
||||
/// dict's file list — i.e. the `btih-v1-file` CID's file index. Serialized into
|
||||
/// the SDK `filelist` redb cache so fan-out survives across `compute_outcomes`
|
||||
/// calls without re-hitting the DHT.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct TorrentFile {
|
||||
pub index: usize,
|
||||
/// File path/name inside the torrent (forward-slash joined).
|
||||
pub name: String,
|
||||
/// File length in bytes.
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// Video file extensions we fan out to (lowercase, no dot). Matches the torznab
|
||||
/// feeder's `VIDEO_EXTENSIONS` so both feeders agree on what "a video file" is.
|
||||
const VIDEO_EXTENSIONS: &[&str] = &[
|
||||
"mkv", "mp4", "avi", "m4v", "mov", "webm", "mpg", "mpeg", "wmv", "flv", "ogv", "ts", "mts",
|
||||
"m2ts",
|
||||
];
|
||||
|
||||
/// True if `name`'s extension is a known video container.
|
||||
pub fn looks_like_video(name: &str) -> bool {
|
||||
name.rsplit('.')
|
||||
.next()
|
||||
.map(|ext| {
|
||||
let lower = ext.to_ascii_lowercase();
|
||||
VIDEO_EXTENSIONS.iter().any(|e| *e == lower)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Tribler `GetMetainfoResponse.files[]` entry. Tolerant field naming: Tribler
|
||||
/// uses `name`/`size`; we also accept `path`/`length` in case a tag differs.
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct MiniFileInfo {
|
||||
#[serde(default)]
|
||||
index: Option<usize>,
|
||||
#[serde(default)]
|
||||
name: Option<String>,
|
||||
#[serde(default)]
|
||||
path: Option<String>,
|
||||
#[serde(default)]
|
||||
size: Option<u64>,
|
||||
#[serde(default)]
|
||||
length: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct MetainfoResponse {
|
||||
#[serde(default)]
|
||||
files: Vec<MiniFileInfo>,
|
||||
}
|
||||
|
||||
/// Parse a Tribler `GetMetainfoResponse` body into [`TorrentFile`]s, filling a
|
||||
/// fallback `index` from array position when absent.
|
||||
pub fn parse_metainfo(body: &str) -> Vec<TorrentFile> {
|
||||
let Ok(resp) = serde_json::from_str::<MetainfoResponse>(body) else {
|
||||
return Vec::new();
|
||||
};
|
||||
resp.files
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(pos, f)| {
|
||||
let name = f.name.or(f.path)?;
|
||||
if name.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(TorrentFile {
|
||||
index: f.index.unwrap_or(pos),
|
||||
name,
|
||||
size: f.size.or(f.length).unwrap_or(0),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn looks_like_video_matches_extensions() {
|
||||
assert!(looks_like_video("Show.S01E01.1080p.mkv"));
|
||||
assert!(looks_like_video("movie.MP4"));
|
||||
assert!(!looks_like_video("readme.txt"));
|
||||
assert!(!looks_like_video("Pressrelease.pdf"));
|
||||
assert!(!looks_like_video("noext"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_metainfo_real_tribler_shape() {
|
||||
// Captured from the live Tribler `/api/torrentinfo/uri` response.
|
||||
let body = r#"{"files":[
|
||||
{"index":0,"name":"Big_Buck_Bunny_1080p.avi","size":928670754},
|
||||
{"index":1,"name":"PROMOTE.txt","size":5008},
|
||||
{"index":2,"name":"Pressrelease.pdf","size":3456234},
|
||||
{"index":3,"name":"license.txt","size":180}
|
||||
],"name":"Big Buck Bunny","description":""}"#;
|
||||
let files = parse_metainfo(body);
|
||||
assert_eq!(files.len(), 4);
|
||||
assert_eq!(files[0].index, 0);
|
||||
assert_eq!(files[0].name, "Big_Buck_Bunny_1080p.avi");
|
||||
assert_eq!(files[0].size, 928670754);
|
||||
let videos: Vec<_> = files.iter().filter(|f| looks_like_video(&f.name)).collect();
|
||||
assert_eq!(videos.len(), 1, "only the .avi is video");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_metainfo_index_fallback_to_position() {
|
||||
let body = r#"{"files":[{"name":"a.mkv","size":1},{"name":"b.mkv","size":2}]}"#;
|
||||
let files = parse_metainfo(body);
|
||||
assert_eq!(files[0].index, 0);
|
||||
assert_eq!(files[1].index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_metainfo_empty_or_garbage() {
|
||||
assert!(parse_metainfo("not json").is_empty());
|
||||
assert!(parse_metainfo(r#"{"files":[]}"#).is_empty());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,63 @@
|
||||
//! Feeder HTTP contract smoke test for `tribler-feeder`.
|
||||
//!
|
||||
//! Boots the real `TriblerPlugin` inside the SDK's `serve_feeders` harness (via
|
||||
//! `router` + `configure_plugins`, so we can bind an ephemeral port) and asserts
|
||||
//! the static surface: `GET /manifest` advertises the `tribler` upstream, and
|
||||
//! `GET /health` is `ok`. Live search/compute are not driven here — they need a
|
||||
//! running Tribler core (IPv8 swarm); that path is exercised by the dev stack's
|
||||
//! bats suite against a real `tribler-instance`.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use meta_feeder_sdk::serve::{configure_plugins, router};
|
||||
use tribler_feeder::tribler::TriblerPlugin;
|
||||
|
||||
async fn boot_feeder() -> (SocketAddr, tempfile::TempDir) {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let plugin = TriblerPlugin::new();
|
||||
let configured =
|
||||
configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure");
|
||||
let app = router(configured, "test".to_string(), dir.path());
|
||||
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind");
|
||||
let addr = listener.local_addr().expect("addr");
|
||||
tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.expect("serve");
|
||||
});
|
||||
(addr, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manifest_advertises_tribler_and_health_ok() {
|
||||
let (addr, _dir) = boot_feeder().await;
|
||||
let base = format!("http://{addr}");
|
||||
let http = reqwest::Client::new();
|
||||
|
||||
let manifest: serde_json::Value = http
|
||||
.get(format!("{base}/manifest"))
|
||||
.send()
|
||||
.await
|
||||
.expect("manifest req")
|
||||
.json()
|
||||
.await
|
||||
.expect("manifest json");
|
||||
let ids: Vec<&str> = manifest["plugins"]
|
||||
.as_array()
|
||||
.expect("plugins array")
|
||||
.iter()
|
||||
.filter_map(|p| p["id"].as_str())
|
||||
.collect();
|
||||
assert!(ids.contains(&"tribler"), "manifest must advertise tribler, got {ids:?}");
|
||||
|
||||
let health: serde_json::Value = http
|
||||
.get(format!("{base}/health"))
|
||||
.send()
|
||||
.await
|
||||
.expect("health req")
|
||||
.json()
|
||||
.await
|
||||
.expect("health json");
|
||||
assert_eq!(health["status"], "ok");
|
||||
}
|
||||
Reference in New Issue
Block a user