inital commit

feeder-plugin/tribler-feeder/src/consts.rs

@@ -0,0 +1,209 @@
+//! Shared constants for the torznab plugin module.
+//!
+//! These symbols are referenced across `mod.rs`, `tmdb.rs` and `enrich.rs`
+//! but were never committed when the plugin was split from a single file into
+//! this directory module — the crate failed to compile (`E0425: cannot find
+//! value ...`). Values below are sensible defaults; review them against the
+//! intended torznab/TMDB contract (endpoints + env-var names especially).
+
+use std::time::Duration;
+
+/// User-Agent for outbound HTTP from the torznab plugin (TMDB + indexers).
+/// Mirrors the per-plugin convention in `plugins/{arxiv,giphy,gutenberg}.rs`.
+pub(crate) const USER_AGENT: &str = concat!(
+    "meta-share/",
+    env!("CARGO_PKG_VERSION"),
+    " (gateway:torznab)"
+);
+
+/// TMDB REST API base (v3).
+pub(crate) const TMDB_API_BASE: &str = "https://api.themoviedb.org/3";
+/// TMDB image CDN base; callers append a size segment (e.g. `/w500/<path>`).
+pub(crate) const TMDB_IMAGE_BASE: &str = "https://image.tmdb.org/t/p";
+/// TMDB poster size segment inserted between [`TMDB_IMAGE_BASE`] and the
+/// poster path. Mandatory — TMDB 404s a path with no size. `w500` is the
+/// poster-grid sweet spot (≈122 KB vs `original`'s multi-MB).
+pub(crate) const TMDB_POSTER_SIZE: &str = "w500";
+
+/// Generic outbound HTTP timeout (seconds). Raised 100x (30 -> 3000) for
+/// completeness — the gateway runs inside a long, streaming search, not a
+/// sub-5s one; an outbound call that's still making progress shouldn't be cut.
+pub(crate) const HTTP_TIMEOUT_SECS: u64 = 3000;
+/// TMDB search request timeout (seconds). Raised 100x (10 -> 1000) — TMDB
+/// cross-ref enrichment is part of result completeness and needs time.
+pub(crate) const TMDB_SEARCH_TIMEOUT_SECS: u64 = 1000;
+/// TMDB poster download timeout (seconds). Raised 100x (30 -> 3000).
+pub(crate) const TMDB_POSTER_TIMEOUT_SECS: u64 = 3000;
+/// Default Retry-After (seconds) applied when TMDB returns HTTP 429 with no
+/// (or an unparseable) `Retry-After` header.
+pub(crate) const TMDB_RATE_LIMIT_DEFAULT_RETRY_SECS: u64 = 5;
+/// Maximum time to wait for a TMDB enrichment permit before giving up (seconds).
+/// Raised 100x (30 -> 3000) — under a deep search many records contend for the
+/// 4 enrichment permits; giving up early drops enrichment from the tail.
+pub(crate) const TMDB_ENRICH_WAIT_DEADLINE_SECS: u64 = 3000;
+/// Maximum number of concurrent TMDB enrichment requests.
+pub(crate) const TMDB_ENRICH_CONCURRENCY: usize = 4;
+/// Per-indexer torznab query timeout (seconds). This is the cap that actually
+/// guillotines slow trackers on the primary *streaming* gateway path (the 5s
+/// `GATEWAY_CALL_TIMEOUT` only bounds the one-shot fallback). Raised 100x
+/// (3 -> 300) for completeness so a slow-but-alive indexer is waited for
+/// rather than silently dropped from the merge. Stays <= the (also-raised)
+/// 500s one-shot `GATEWAY_CALL_TIMEOUT`. NOTE: `search_probe_timeout()` reuses
+/// this value, so reachability probes against a *dead* indexer now hang up to
+/// 300s — acceptable for a completeness-first posture, but the dual use is why
+/// this isn't split into its own probe constant yet.
+pub(crate) const PER_INDEXER_QUERY_TIMEOUT_SECS: u64 = 300;
+
+/// Max concurrent `.torrent` downloads while resolving one indexer's results.
+/// Bounds the socket burst when a whole result page is torrent-only. Kept low
+/// (4) because the resolve fetches hit public trackers/torrent caches that
+/// reset connections under a wide burst — the same load that makes Prowlarr's
+/// own indexer calls flake. Indexer *search* GETs are now serialized one at a
+/// time through the shared [`INDEXER_LIMITER_RPS`] token bucket, so the peak
+/// socket count is just this many (the in-flight page's resolves), not a
+/// product across concurrent indexers.
+pub(crate) const TORRENT_RESOLVE_CONCURRENCY: usize = 4;
+
+/// Defensive cap on how many torrent-only items we attempt to resolve per
+/// indexer response. The rest are dropped (logged) — at search latency,
+/// resolving an unbounded page of `.torrent` files would blow the per-indexer
+/// budget. A page rarely needs more than this many resolved results anyway.
+pub(crate) const TORRENT_RESOLVE_MAX_PER_INDEXER: usize = 30;
+
+/// Wall-clock budget for the whole torrent-resolution phase of a single
+/// indexer call. Items not resolved within it are dropped; items resolved so
+/// far are kept (partial). The gateway returns as soon as resolution
+/// *completes* — the budget is only the cutoff for a page that can't finish.
+///
+/// **Sized for a long, streaming search, not a sub-5s one.** meta-share's
+/// primary gateway path is a streaming substream (`stream_one_gateway_peer`)
+/// with **no per-call deadline** — it forwards each hit to the client SSE as
+/// it arrives until the gateway sends `Done`. (The 5 s `GATEWAY_CALL_TIMEOUT`
+/// is only the one-shot fallback for gateways that don't speak the streaming
+/// protocol.) So this can be generous: the binding constraint is meta-share's
+/// `SEARCH_TIMEOUT_SECS` silence window (set to match in the dev compose), not
+/// a hard call timeout. 5 min is a safety ceiling on a pathological page;
+/// a normal XXX page (≤ `TORRENT_RESOLVE_MAX_PER_INDEXER` items at ~1.9 s
+/// each, 8-wide) finishes in well under 10 s and the gateway returns then.
+pub(crate) const TORRENT_RESOLVE_BUDGET: Duration = Duration::from_secs(300);
+
+/// Ceiling on a downloaded `.torrent` body. Real torrent files are KBs; this
+/// guards against a `<link>` that points at an HTML details page or a
+/// mislabeled giant download — we skip (don't buffer) anything larger.
+pub(crate) const TORRENT_FILE_MAX_BYTES: u64 = 5 * 1024 * 1024;
+
+/// Magnet BEP-9 `list_only` file-list fallback knobs (packs only) — see
+/// [`crate::bt::TorznabBt::prefetch_torrent_filelists`].
+///
+/// HTTP `.torrent` download is always tried first (faster, swarm-free). But
+/// magnet-only indexers (e.g. Nyaa.si) expose no `.torrent` URL, so a season
+/// **pack** from them never gets a file list and its per-episode records ship
+/// without a streamable `cid_btih_v1_file` (see `expand_record_streaming`).
+/// For packs only — a single file derives its CID from the infohash alone and
+/// needs no file list — fall back to a bounded BEP-9 `list_only` swarm probe
+/// off the magnet. Swarm peer/DHT discovery is far slower (and often fruitless
+/// for dead torrents) than an HTTP fetch, so these caps are deliberately tight:
+/// the cached result makes the *next* search instant, so a partial pass is fine.
+pub(crate) const TORRENT_LISTONLY_MAGNET_MAX: usize = 6;
+/// Concurrent magnet enumerations. Lower than [`TORRENT_RESOLVE_CONCURRENCY`]
+/// because each holds a live librqbit torrent + peer connections, not a single
+/// short HTTP GET.
+pub(crate) const TORRENT_LISTONLY_MAGNET_CONCURRENCY: usize = 2;
+/// Per-magnet wall-clock cap on the BEP-9 enumeration (tighter than
+/// `list_files`' own internal full-fetch budget, which is sized for downloads).
+pub(crate) const TORRENT_LISTONLY_MAGNET_PER_ITEM: Duration = Duration::from_secs(25);
+/// Overall deadline for the whole magnet-fallback pass; whatever resolved by
+/// then is kept, the rest fall through to the title-only (un-CID'd) expansion.
+pub(crate) const TORRENT_LISTONLY_MAGNET_BUDGET: Duration = Duration::from_secs(60);
+
+/// **Test-only.** When truthy (`1`/`true`/`yes`/`on`), a torznab record that
+/// fails TMDB enrichment is stubbed with just enough metadata (a `poster`, a
+/// `description`, and video kind if missing) to pass meta-watch's quality
+/// gate — simulating "content pollution" so an operator can validate the
+/// default-on adult-content filter against records that carry `categories/XXX`
+/// but never match TMDB. **Off by default; never enable in production** — it
+/// fabricates metadata. See [`super::enrich::stub_unmatched_patch`].
+pub(crate) const ENV_STUB_UNMATCHED: &str = "META_GATEWAY_PROWLARR_STUB_UNMATCHED";
+
+/// Timeout used when probing an indexer's search path for reachability.
+pub(crate) fn search_probe_timeout() -> Duration {
+    Duration::from_secs(PER_INDEXER_QUERY_TIMEOUT_SECS)
+}
+
+/// Default number of TMDB anchors resolved per query (top-N by popularity).
+/// For a vague keyword ("black") TMDB returns many real shows; we anchor on the
+/// top-N rather than collapsing onto the single most popular (the old
+/// `principal_confident` behaviour that hijacked the query). Each anchor becomes
+/// its own paged search per indexer, so this multiplies the indexer request
+/// count — the serialized [`INDEXER_LIMITER_RPS`] bucket keeps that gentle.
+/// Overridable via `META_GATEWAY_PROWLARR_ANCHOR_TOP_N`.
+pub(crate) const DEFAULT_ANCHOR_TOP_N: usize = 10;
+
+/// How many ranked principal-search anchors to persist per query in the redb
+/// cache. Sized comfortably above any reasonable top-N so flipping the knob
+/// never needs a cache wipe — reads slice the cached list to the configured N.
+pub(crate) const CACHED_PRINCIPAL_DEPTH: usize = 30;
+
+/// Indexer search-request rate limiter (a [`crate::tmdb_budget::TmdbBudget`]
+/// token bucket shared across the whole torznab fan-out). Public trackers and
+/// Prowlarr reset/429 under bursts, so search GETs are serialized one at a time
+/// at ~`INDEXER_LIMITER_RPS` grants/sec with a small `INDEXER_LIMITER_BURST`.
+/// Adaptive: a 429 (or transient reset/5xx) pauses the bucket via `note_429`.
+pub(crate) const INDEXER_LIMITER_RPS: f64 = 1.0;
+/// Burst capacity of the indexer rate limiter (see [`INDEXER_LIMITER_RPS`]).
+pub(crate) const INDEXER_LIMITER_BURST: f64 = 3.0;
+/// Backoff applied to the indexer limiter on a transient (connection-reset /
+/// 5xx) indexer failure that carries no explicit `Retry-After` window.
+pub(crate) const INDEXER_LIMITER_TRANSIENT_BACKOFF: Duration = Duration::from_secs(2);
+/// Deadline for acquiring an indexer-limiter token before a page request.
+/// Generous (completeness-first; meta-share waits its silence window) so a
+/// token effectively always arrives — `DeadlineExceeded` just drops that page.
+pub(crate) const INDEXER_LIMITER_WAIT_DEADLINE_SECS: u64 = 3000;
+
+/// Minimum fraction of a (cleaned) release title's tokens that must be covered
+/// by an anchor's title before a text-anchored job stamps that anchor's tmdbid
+/// onto the release. Guards against text false-positives — a release that
+/// merely *mentions* the anchor title (e.g. "Turning Mecard … Black Mirror",
+/// 2/≈9 ≈ 0.22) stays unanchored and enriches via its own fuzzy lookup, while a
+/// real "Black Mirror S05E01 …" (cleans to "Black Mirror", 2/2 = 1.0) is
+/// stamped. Only applies to fuzzy text jobs; id-based (tvdbid/tmdbid) jobs are
+/// trusted and bypass the guard. See [`super::anchor::apply_record_tag`].
+pub(crate) const ANCHOR_TITLE_MIN_COVERAGE: f64 = 0.6;
+
+/// Backoff before the single retry [`super::TorznabPlugin::send_get_with_retry`]
+/// makes after a transient (connection-reset / 5xx) indexer failure. Short —
+/// just enough to let a momentarily-overloaded indexer or a stale pooled
+/// connection clear before the second attempt.
+pub(crate) const HTTP_RETRY_BACKOFF: Duration = Duration::from_millis(400);
+
+/// Defensive ceiling on the per-season fan-out loop. A show with more seasons
+/// than this (none real; guards a corrupt `number_of_seasons`) only fans out
+/// the first `MAX_ANCHORED_SEASONS`.
+pub(crate) const MAX_ANCHORED_SEASONS: u32 = 50;
+
+/// Per-indexer fetch page size for the depth strategy. Clamped to each
+/// indexer's declared Prowlarr `capabilities.limitsMax`. Kept small/light so a
+/// single Prowlarr `indexerIds=<one>` call behaves like the Prowlarr UI (one
+/// tracker, a modest page) instead of one heavy `indexerIds=-2&limit=2500`
+/// aggregate — that oversized limit was both futile (every public tracker caps
+/// at ~100) and the source of the gateway→Prowlarr connection resets under load.
+pub(crate) const TORZNAB_PAGE_SIZE: usize = 100;
+
+/// Bounded buffer between the streaming search producer task and the SSE-bound
+/// receiver stream. Backpressure: when the consumer (meta-share) is slow the
+/// producer's `send().await` parks rather than piling events in memory.
+pub(crate) const GATEWAY_STREAM_BUFFER: usize = 256;
+
+/// Max results fetched **per indexer, per search** (depth cap). Bounds both the
+/// single-shot request for a non-paginating indexer and the offset-paging loop
+/// for one that supports pagination. Tunable; the merge across indexers + the
+/// caller's display `limit` truncate still apply on top. With today's 5 public
+/// trackers (all `limitsMax=100`, no pagination) the effective per-indexer
+/// fetch is 100, so 5×100 ≈ 500 merged — which is where this default lands.
+pub(crate) const TORZNAB_MAX_DEPTH: usize = 500;
+
+/// Ceiling on how many records one torrent may fan out to when a pack is
+/// unpacked into per-file episodes (or range-inferred). A guard against a
+/// pathological 1000-file torrent flooding one search; well above any real
+/// season pack. Mirrors the tribler feeder's identical cap.
+pub(crate) const MAX_FANOUT_FILES: usize = 100;