diff --git a/Cargo.toml b/Cargo.toml
index 023569c..68974d6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
[workspace]
resolver = "2"
-members = ["crates/*", "feeder-plugin/*"]
+members = ["feeder-plugin/*"]
[workspace.package]
edition = "2021"
diff --git a/README.md b/README.md
index 2f084a5..8b1bacf 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,13 @@ credential anywhere in the path**.
This feeder **adds** a Usenet metadata source alongside `meta-feeder-indexer`.
It replaces nothing; a gateway query fans out to both.
+It also **redeems external releases.** The torznab feeder mints `nzb-release`
+(`0x1005`) locators from Prowlarr/Newznab search hits; when a viewer plays one,
+the gateway asks this feeder (`POST /compute` with the locator cid) to grab its
+`.nzb` with the key configured here for that indexer host. The Newznab keys live
+on this feeder — meta-share keeps only the NNTP provider. See
+[Redeeming nzb-release locators](#redeeming-nzb-release-locators).
+
Design record:
`meta-gateway/docs/others/self-hosted-usenet-indexer-study.md`.
@@ -61,8 +68,12 @@ and there is no hot reload.
| Field | Env seed | What |
|---|---|---|
-| `db_url` | `NNTMUX_DB_URL` | MariaDB DSN of nntmux's catalog, e.g. `mysql://nntmux:secret@nntmux-db:3306/nntmux`. Blank → the upstream soft-skips and the plugin reports Degraded. |
+| `db_url` | `NNTMUX_DB_URL` | MariaDB DSN of nntmux's catalog, e.g. `mysql://nntmux:secret@nntmux-db:3306/nntmux`. Blank → self-scan search is off. |
| `nzb_root` | `NNTMUX_NZB_PATH` | nntmux's NZB directory **as mounted into this container**. Must be the same volume nntmux's `PATH_TO_NZBS` points at, or every release resolves to `NotFound`. |
+| `indexers` | — (config page only) | `[{host, api_key}]` Newznab keys for redeeming `nzb-release` locators. `host` is the bare authority (`api.nzbgeek.info` — no scheme, no `/api`). |
+
+Both halves are optional; with neither configured the plugin reports Degraded.
+A grab-only deployment (keys, no nntmux) is valid.
Infra env: `META_FEEDER_HTTP_LISTEN` (default `0.0.0.0:8080`),
`META_FEEDER_STATE_DIR` (default `/data/meta-feeder`), `RUST_LOG`.
@@ -73,6 +84,32 @@ Keep the connection budget conservative: the scanner and playback draw on the
same provider slots, so a greedy value does not merely over-scan, it stalls
playback mid-stream.
+## Redeeming nzb-release locators
+
+`POST /compute {upstream_id: "usenet", record_id: <0x1005 cid>}`:
+
+1. Decode `{api_base, id}` from the cid; find the key for its host. No key →
+ `404` (not ours — the gateway tries another claimer, nothing is spent).
+2. `GET https://{api_base}/api?t=get&id=…&apikey=…` with a named User-Agent
+ (nzbgeek answers `109 Invalid User Agent` without one). **This spends the
+ indexer's daily download quota** — the gateway only calls it on a real play,
+ and never again for a release whose `manifest` it already stored.
+3. Strip the `.nzb` `` except `` (the file is seeded
+ to the whole mesh; the password is needed to unpack a protected RAR set).
+4. Mint the portable `nzb-posting` (`0x1003`) cid from its Message-IDs.
+5. Answer ONE `sha2_256` outcome: the `.nzb` bytes (`file_extension: nzb`) and a
+ record carrying `cids/` + `segmentCount`. The gateway stores the
+ file under `/files/plugin/meta-feeder-usenet/`, writes its cid onto the release
+ record as `manifest`, and merges the posting cid — after which any peer with a
+ plain NNTP provider can play the release.
+
+Errors: an indexer credential/account refusal (`100`–`102`, `910`) or an HTML
+page → `422`; request/download limit (`429`, `500`, `501`, or HTTP 429) → `429`;
+no such item (`300`) → `404`.
+
+`GET /redeems` and `/manifest` advertise `{codec: nzb-release, field: manifest,
+hosts: [...]}` for the hosts that have a key; nothing without keys.
+
## Build
```bash
@@ -83,11 +120,26 @@ cargo clippy --workspace --all-targets -- -D warnings
docker build -f feeder-plugin/usenet-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-usenet:dev .
```
-`crates/meta-feeder-sdk/` is a **vendored copy** of the SDK, mirrored from
-`meta-gateway`. Independent submodules cannot share a cargo path-dep, so keep it
-in step when the SDK changes — in particular `hash.rs`'s
-`compute_nzb_posting_cid`, whose normalisation rule is the dedup contract and is
-mirrored again on the meta-share side.
+`meta-feeder-sdk` is a git dependency pinned by tag (`v1.3.0`; this repo used to
+vendor a copy). Until that tag is pushed, cargo cannot resolve it — not even
+under a `[patch]` override. To verify against a local SDK checkout, build a
+scratch copy with the dependency swapped for a path (never commit that form —
+this repo is the Docker build context):
+
+```bash
+SCRATCH=/d/workspace/tmp-claude/usenet-feeder; SDK=/d/workspace/MetaMesh/meta-root-v2/packages/plugins/meta-feeder-sdk
+rm -rf $SCRATCH && mkdir -p $SCRATCH && tar cf - --exclude=./target --exclude=./.git . | (cd $SCRATCH && tar xf -)
+sed -i "s#meta-feeder-sdk = { git = .*#meta-feeder-sdk = { path = \"$SDK\" }#" $SCRATCH/feeder-plugin/*/Cargo.toml
+docker run --rm -e RUSTUP_TOOLCHAIN=1.89.0 -e CARGO_TARGET_DIR=/target -v /d/workspace:/d/workspace \
+ -v feeder-cargo-registry:/usr/local/cargo/registry -v usenet-feeder-target:/target -w $SCRATCH \
+ rust:1.89 sh -c 'apt-get install -y -qq pkg-config libssl-dev >/dev/null 2>&1; cargo test'
+```
+
+After the tag is pushed, run `cargo update -p meta-feeder-sdk` here so
+`Cargo.lock` stops pointing at the removed vendored path.
+
+`compute_nzb_posting_cid`'s normalisation rule is the dedup contract and is
+mirrored on the meta-share side — change them together.
## Gotchas
diff --git a/crates/meta-feeder-sdk/Cargo.toml b/crates/meta-feeder-sdk/Cargo.toml
deleted file mode 100644
index bbe98ff..0000000
--- a/crates/meta-feeder-sdk/Cargo.toml
+++ /dev/null
@@ -1,49 +0,0 @@
-[package]
-name = "meta-feeder-sdk"
-version = "0.1.0"
-edition.workspace = true
-rust-version.workspace = true
-license.workspace = true
-publish = false
-
-# Source-agnostic foundation shared by the gateway core and every feeder
-# sidecar binary: the `FeederPlugin` trait, the content-addressing hash
-# families, the per-plugin redb cache, the shared HTTP/cache helpers, the
-# structured-query evaluator, and the axum `serve_feeders` harness that wraps
-# any set of `FeederPlugin`s in the feeder HTTP contract.
-#
-# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes;
-# the gateway core owns hashing-into-the-blockstore, the meta-core store-back,
-# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT
-# of here so feeders that don't need them don't pay for them.
-
-[dependencies]
-# Async runtime + HTTP server (the serve_feeders harness)
-tokio = { version = "1.40", features = ["full"] }
-axum = { version = "0.7", features = ["json"] }
-hyper = { version = "1", features = ["full"] }
-reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
-
-# Serialization
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-base64 = "0.22"
-
-# Utilities
-futures = "0.3"
-async-trait = "0.1"
-anyhow = "1"
-thiserror = "1"
-tracing = "0.1"
-sha2 = "0.10"
-bytes = "1"
-# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
-regex = "1"
-
-# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
-# cache.rs. Pure-Rust single-process embedded store.
-redb = "2"
-
-[dev-dependencies]
-tempfile = "3"
-tokio = { version = "1.40", features = ["full", "test-util"] }
diff --git a/crates/meta-feeder-sdk/src/cache.rs b/crates/meta-feeder-sdk/src/cache.rs
deleted file mode 100644
index e04aae4..0000000
--- a/crates/meta-feeder-sdk/src/cache.rs
+++ /dev/null
@@ -1,353 +0,0 @@
-//! Per-plugin midhash cache backed by redb.
-//!
-//! Each plugin gets its own redb file at
-//! `/gateway//cache.redb`. Keys are upstream
-//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the
-//! `Midhash` strings the plugin computed by fetching + hashing the upstream
-//! file.
-//!
-//! The schema is identical to meta-share v1's so old `.redb` files copy
-//! across. redb is synchronous and sub-millisecond for small ops, so calls
-//! run inline from async handlers without `spawn_blocking`.
-
-use std::path::Path;
-use std::sync::Arc;
-
-use redb::{Database, ReadableTableMetadata, TableDefinition};
-
-pub const CACHE_FILENAME: &str = "cache.redb";
-
-const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash");
-const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs");
-const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid");
-const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec");
-const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid");
-/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload
-/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the
-/// hash families differ: `midhash` is the synthetic midhash256-from-infohash
-/// fallback; `fullhash` records "we actually downloaded the bytes and this is
-/// the IPFS CIDv1 over them."
-const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash");
-/// Enumerated torrent file list (JSON `Vec`) keyed by the
-/// torrent's `record_id` (infohash).
-const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist");
-/// Cached TMDB *search* results keyed by a stable lookup key. The value is the
-/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache
-/// sentinel.
-const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search");
-/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`.
-const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> =
- TableDefinition::new("tmdb_tvdetails");
-/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`.
-const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids");
-/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`.
-const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> =
- TableDefinition::new("tmdb_moviedetails");
-/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized
-/// free-text query.
-const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> =
- TableDefinition::new("tmdb_principal_topn");
-/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's
-/// `record_id` (infohash). Value is the JSON encoding of a `Vec`.
-const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles");
-/// OpenSubtitles search results keyed by `"\x01"`. Value is the
-/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel.
-const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles");
-/// Small durable plugin scratch values that must survive a restart but aren't
-/// content-derived — today, the torznab plugin's Prowlarr indexer-set
-/// fingerprint, so a set that changed *while the feeder was down* is still
-/// detected as a change on the next boot. Keys are plugin-namespaced
-/// (`prowlarr:indexer-set`). redb creates the table on open, so an older
-/// `.redb` file gains it transparently.
-const MISC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("misc");
-
-/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared
-/// via `Arc`).
-#[derive(Clone)]
-pub struct MidhashCache {
- db: Arc,
-}
-
-/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb
-/// table.
-macro_rules! str_table_accessors {
- ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => {
- $(#[$gmeta])*
- pub fn $get(&self, key: &str) -> Result, redb::Error> {
- let tx = self.db.begin_read()?;
- let table = tx.open_table($table)?;
- Ok(table.get(key)?.map(|v| v.value().to_string()))
- }
-
- $(#[$pmeta])*
- pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> {
- let tx = self.db.begin_write()?;
- {
- let mut table = tx.open_table($table)?;
- table.insert(key, value)?;
- }
- tx.commit()?;
- Ok(())
- }
- };
-}
-
-impl MidhashCache {
- pub fn open(cache_dir: &Path) -> Result {
- let path = cache_dir.join(CACHE_FILENAME);
- let db = Database::create(&path)?;
- {
- let tx = db.begin_write()?;
- tx.open_table(MIDHASH_TABLE)?;
- tx.open_table(BLOBS_TABLE)?;
- tx.open_table(COVER_CID_TABLE)?;
- tx.open_table(BIBREC_TABLE)?;
- tx.open_table(PREVIEW_CID_TABLE)?;
- tx.open_table(FULLHASH_TABLE)?;
- tx.open_table(FILELIST_TABLE)?;
- tx.open_table(TMDB_SEARCH_TABLE)?;
- tx.open_table(TMDB_TVDETAILS_TABLE)?;
- tx.open_table(TMDB_EXTIDS_TABLE)?;
- tx.open_table(TMDB_MOVIEDETAILS_TABLE)?;
- tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?;
- tx.open_table(SUBTITLES_TABLE)?;
- tx.open_table(OPENSUBTITLES_TABLE)?;
- tx.open_table(MISC_TABLE)?;
- tx.commit()?;
- }
- Ok(Self { db: Arc::new(db) })
- }
-
- str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE);
-
- str_table_accessors!(
- /// Read a durable plugin scratch value (see [`MISC_TABLE`]).
- get_misc,
- /// Write a durable plugin scratch value (see [`MISC_TABLE`]).
- put_misc,
- MISC_TABLE,
- );
-
- pub fn entry_count(&self) -> Result {
- let tx = self.db.begin_read()?;
- let table = tx.open_table(MIDHASH_TABLE)?;
- Ok(table.len()?)
- }
-
- pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> {
- let tx = self.db.begin_read()?;
- let table = tx.open_table(BLOBS_TABLE)?;
- Ok(table.get(cid)?.map(|v| v.value().to_vec()))
- }
-
- pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> {
- let tx = self.db.begin_write()?;
- {
- let mut table = tx.open_table(BLOBS_TABLE)?;
- table.insert(cid, bytes)?;
- }
- tx.commit()?;
- Ok(())
- }
-
- str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE);
-
- str_table_accessors!(
- /// Read a previously-stored full-file IPFS cid for `record_id`.
- get_fullhash,
- /// Record the IPFS cid produced by a successful full fetch.
- put_fullhash,
- FULLHASH_TABLE
- );
-
- str_table_accessors!(
- /// Read the cached torrent file list (JSON) for `record_id`.
- get_filelist,
- /// Cache the enumerated torrent file list (JSON `Vec`).
- put_filelist,
- FILELIST_TABLE
- );
-
- str_table_accessors!(
- /// Read the cached subtitle linkage (JSON `Vec`).
- get_subtitles,
- /// Cache the discovered subtitle linkage (JSON `Vec`).
- put_subtitles,
- SUBTITLES_TABLE
- );
-
- str_table_accessors!(
- /// Read a cached OpenSubtitles lookup by `"\x01"`.
- get_opensubtitles,
- /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel).
- put_opensubtitles,
- OPENSUBTITLES_TABLE
- );
-
- str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE);
-
- str_table_accessors!(
- /// Read a cached TMDB search result by lookup key.
- get_tmdb_search,
- /// Cache a TMDB search result under `key`.
- put_tmdb_search,
- TMDB_SEARCH_TABLE
- );
-
- str_table_accessors!(
- /// Read cached TMDB TV-details JSON by `tmdbid`.
- get_tmdb_tvdetails,
- /// Cache TMDB TV-details JSON under `tmdbid`.
- put_tmdb_tvdetails,
- TMDB_TVDETAILS_TABLE
- );
-
- str_table_accessors!(
- /// Read cached TMDB external-ids JSON by `tmdbid`.
- get_tmdb_extids,
- /// Cache TMDB external-ids JSON under `tmdbid`.
- put_tmdb_extids,
- TMDB_EXTIDS_TABLE
- );
-
- str_table_accessors!(
- /// Read cached TMDB movie-details JSON by `tmdbid`.
- get_tmdb_moviedetails,
- /// Cache TMDB movie-details JSON under `tmdbid`.
- put_tmdb_moviedetails,
- TMDB_MOVIEDETAILS_TABLE
- );
-
- str_table_accessors!(
- /// Read the cached ranked anchor list by normalized query key.
- get_tmdb_principal_topn,
- /// Cache the ranked anchor list under the normalized query key.
- put_tmdb_principal_topn,
- TMDB_PRINCIPAL_TOPN_TABLE
- );
-
- pub fn get_bibrec(
- &self,
- record_id: &str,
- ) -> Result>, redb::Error> {
- let tx = self.db.begin_read()?;
- let table = tx.open_table(BIBREC_TABLE)?;
- let Some(v) = table.get(record_id)? else {
- return Ok(None);
- };
- let s = v.value();
- Ok(serde_json::from_str(s).ok())
- }
-
- pub fn put_bibrec(
- &self,
- record_id: &str,
- fields: &std::collections::BTreeMap,
- ) -> Result<(), redb::Error> {
- let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string());
- let tx = self.db.begin_write()?;
- {
- let mut table = tx.open_table(BIBREC_TABLE)?;
- table.insert(record_id, json.as_str())?;
- }
- tx.commit()?;
- Ok(())
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn tmp_cache_dir() -> tempfile::TempDir {
- tempfile::tempdir().expect("tempdir")
- }
-
- #[test]
- fn put_then_get_returns_value() {
- let dir = tmp_cache_dir();
- let cache = MidhashCache::open(dir.path()).expect("open");
- assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None);
- cache.put_midhash("10.1038/x", "bafyXYZ").expect("put");
- assert_eq!(
- cache.get_midhash("10.1038/x").unwrap().as_deref(),
- Some("bafyXYZ")
- );
- }
-
- #[test]
- fn put_overwrites_existing_value() {
- let dir = tmp_cache_dir();
- let cache = MidhashCache::open(dir.path()).expect("open");
- cache.put_midhash("k", "v1").unwrap();
- cache.put_midhash("k", "v2").unwrap();
- assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2"));
- }
-
- #[test]
- fn values_persist_across_reopens() {
- let dir = tmp_cache_dir();
- {
- let cache = MidhashCache::open(dir.path()).expect("open 1");
- cache.put_midhash("k", "persistent").unwrap();
- }
- let cache = MidhashCache::open(dir.path()).expect("open 2");
- assert_eq!(
- cache.get_midhash("k").unwrap().as_deref(),
- Some("persistent")
- );
- }
-
- #[test]
- fn tmdb_search_cache_distinguishes_miss_from_absent() {
- let dir = tmp_cache_dir();
- let cache = MidhashCache::open(dir.path()).expect("open");
- assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None);
- cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap();
- assert_eq!(
- cache
- .get_tmdb_search("m\u{1}naruto\u{1}")
- .unwrap()
- .as_deref(),
- Some("null")
- );
- cache
- .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}")
- .unwrap();
- drop(cache);
- let cache = MidhashCache::open(dir.path()).expect("reopen");
- assert_eq!(
- cache
- .get_tmdb_search("t\u{1}spy x family\u{1}2019")
- .unwrap()
- .as_deref(),
- Some("{\"tmdbid\":120089}")
- );
- }
-
- #[test]
- fn tmdb_tvdetails_cache_roundtrips() {
- let dir = tmp_cache_dir();
- let cache = MidhashCache::open(dir.path()).expect("open");
- assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None);
- cache
- .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}")
- .unwrap();
- assert_eq!(
- cache.get_tmdb_tvdetails("120089").unwrap().as_deref(),
- Some("{\"number_of_seasons\":2}")
- );
- }
-
- #[test]
- fn entry_count_reflects_inserts() {
- let dir = tmp_cache_dir();
- let cache = MidhashCache::open(dir.path()).expect("open");
- assert_eq!(cache.entry_count().unwrap(), 0);
- cache.put_midhash("a", "x").unwrap();
- cache.put_midhash("b", "y").unwrap();
- assert_eq!(cache.entry_count().unwrap(), 2);
- cache.put_midhash("a", "x2").unwrap();
- assert_eq!(cache.entry_count().unwrap(), 2);
- }
-}
diff --git a/crates/meta-feeder-sdk/src/common.rs b/crates/meta-feeder-sdk/src/common.rs
deleted file mode 100644
index c7a09f5..0000000
--- a/crates/meta-feeder-sdk/src/common.rs
+++ /dev/null
@@ -1,288 +0,0 @@
-//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`,
-//! `pubmed`, `wikicommons`, `giphy`, `scihub`).
-//!
-//! These plugins are structurally near-identical: build a reqwest client, map
-//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose
-//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding
-//! lives here; the genuinely-different bodies stay in the plugin modules.
-
-use std::path::Path;
-use std::time::Duration;
-
-use tracing::{debug, warn};
-
-use crate::cache::MidhashCache;
-use crate::plugin::{ConfigError, HashKind, HashOutcome};
-use crate::types::{DiscoveryRecord, GatewayError, Hash};
-
-/// Build the shared reqwest client every source plugin uses.
-pub fn build_http_client(
- timeout_secs: u64,
- user_agent: &str,
- redirect: Option,
-) -> reqwest::Client {
- let mut builder = reqwest::Client::builder()
- .timeout(Duration::from_secs(timeout_secs))
- .user_agent(user_agent);
- if let Some(policy) = redirect {
- builder = builder.redirect(policy);
- }
- builder
- .build()
- .expect("reqwest client build is infallible with rustls-tls")
-}
-
-/// Map a non-success HTTP status to a [`GatewayError`].
-///
-/// - `404` → [`GatewayError::NotFound`]
-/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s)
-/// - `5xx` → [`GatewayError::Transient`]
-/// - anything else → [`GatewayError::Permanent`]
-pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> {
- let status = resp.status();
- if status.is_success() {
- return Ok(());
- }
- let code = status.as_u16();
- if code == 404 {
- return Err(GatewayError::NotFound);
- }
- if code == 429 {
- let retry_after = resp
- .headers()
- .get(reqwest::header::RETRY_AFTER)
- .and_then(|v| v.to_str().ok())
- .and_then(|v| v.parse::().ok())
- .unwrap_or(60);
- return Err(GatewayError::RateLimited {
- retry_after_s: retry_after,
- });
- }
- if status.is_server_error() {
- return Err(GatewayError::Transient(format!(
- "upstream {} returned {status}",
- resp.url()
- )));
- }
- Err(GatewayError::Permanent(format!(
- "upstream {} returned {status}",
- resp.url()
- )))
-}
-
-/// Open the per-plugin redb midhash cache, wrapping the error with the plugin
-/// name for a clean startup diagnostic.
-pub fn open_midhash_cache(
- cache_dir: &Path,
- plugin: &'static str,
-) -> Result {
- MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other {
- plugin,
- source: anyhow::anyhow!("open redb cache: {e}"),
- })
-}
-
-/// Cache-hit fast path for single-outcome plugins.
-///
-/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record —
-/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a
-/// cache miss; the caller proceeds to fetch + hash.
-pub fn cached_outcome(
- cache: &MidhashCache,
- record_id: &str,
- upstream_id: &'static str,
-) -> Result>, GatewayError> {
- let Some(cached) = cache
- .get_midhash(record_id)
- .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))?
- else {
- return Ok(None);
- };
- debug!(
- target: "meta-share::gateway",
- upstream = upstream_id,
- record_id,
- "hash cache hit"
- );
- Ok(Some(vec![HashOutcome {
- hash: Hash(cached),
- hash_kind: HashKind::Sha2_256,
- bytes: None,
- record: None,
- file_extension: None,
- }]))
-}
-
-/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is
-/// non-fatal, so it is logged and swallowed.
-pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) {
- if let Err(e) = cache.put_midhash(record_id, cid) {
- warn!(
- target: "meta-share::gateway",
- upstream = upstream_id,
- record_id,
- error = %e,
- "hash cache put failed (non-fatal)"
- );
- }
-}
-
-/// Resolve the per-plugin cache handle, or produce the standard "not
-/// configured" internal error.
-pub fn require_cache<'a>(
- cache: Option<&'a MidhashCache>,
- plugin: &str,
-) -> Result<&'a MidhashCache, GatewayError> {
- cache.ok_or_else(|| {
- GatewayError::Internal(anyhow::anyhow!(
- "{plugin} plugin not configured (configure() never called)"
- ))
- })
-}
-
-/// Build the single-element outcome vec a single-outcome plugin returns on a
-/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID,
-/// and an optional file extension.
-pub fn single_outcome(
- cid: String,
- bytes: bytes::Bytes,
- record: DiscoveryRecord,
- file_extension: Option,
-) -> Vec {
- vec![HashOutcome {
- hash: Hash(cid),
- hash_kind: HashKind::Sha2_256,
- bytes: Some(bytes),
- record: Some(record),
- file_extension,
- }]
-}
-
-/// Percent-encode a query-string value using the unreserved set
-/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space.
-pub fn urlencode(s: &str) -> String {
- let mut out = String::with_capacity(s.len());
- for b in s.as_bytes() {
- match *b {
- b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
- out.push(*b as char);
- }
- b' ' => out.push('+'),
- other => out.push_str(&format!("%{:02X}", other)),
- }
- }
- out
-}
-
-/// GET `url` and return the body bytes, mapping transport + status errors to
-/// [`GatewayError`].
-pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result {
- let resp = http
- .get(url)
- .send()
- .await
- .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?;
- map_status(&resp)?;
- resp.bytes()
- .await
- .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}")))
-}
-
-/// Like [`fetch_bytes`] but decodes the body as UTF-8 text.
-pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result {
- let resp = http
- .get(url)
- .send()
- .await
- .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?;
- map_status(&resp)?;
- resp.text()
- .await
- .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}")))
-}
-
-/// Decode a v1 BitTorrent infohash string into its raw 20 bytes.
-///
-/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648,
-/// case-insensitive). Returns `None` for anything else.
-pub fn decode_infohash(s: &str) -> Option<[u8; 20]> {
- let bytes = s.as_bytes();
- let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) {
- let mut out = Vec::with_capacity(20);
- for pair in bytes.chunks(2) {
- out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?);
- }
- out
- } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) {
- decode_base32_20(bytes)?
- } else {
- return None;
- };
- raw.try_into().ok()
-}
-
-fn hex_nibble(b: u8) -> Option {
- match b {
- b'0'..=b'9' => Some(b - b'0'),
- b'a'..=b'f' => Some(b - b'a' + 10),
- b'A'..=b'F' => Some(b - b'A' + 10),
- _ => None,
- }
-}
-
-/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20
-/// bytes they encode.
-fn decode_base32_20(chars: &[u8]) -> Option> {
- let mut out = Vec::with_capacity(20);
- let mut buffer: u64 = 0;
- let mut bits: u32 = 0;
- for &c in chars {
- let val = match c {
- b'A'..=b'Z' => c - b'A',
- b'a'..=b'z' => c - b'a',
- b'2'..=b'7' => c - b'2' + 26,
- _ => return None,
- } as u64;
- buffer = (buffer << 5) | val;
- bits += 5;
- if bits >= 8 {
- bits -= 8;
- out.push((buffer >> bits) as u8);
- }
- }
- Some(out)
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn decode_infohash_hex_roundtrips() {
- let hex = "0123456789abcdef0123456789abcdef01234567";
- let raw = decode_infohash(hex).expect("hex decodes");
- assert_eq!(raw[0], 0x01);
- assert_eq!(raw[19], 0x67);
- }
-
- #[test]
- fn decode_infohash_is_case_insensitive_hex() {
- let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd");
- let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD");
- assert!(lower.is_some());
- assert_eq!(lower, upper);
- }
-
- #[test]
- fn decode_infohash_accepts_base32() {
- let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U");
- assert!(raw.is_some());
- }
-
- #[test]
- fn decode_infohash_rejects_wrong_shape() {
- assert!(decode_infohash("tooshort").is_none());
- assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none());
- assert!(decode_infohash("").is_none());
- }
-}
diff --git a/crates/meta-feeder-sdk/src/config.rs b/crates/meta-feeder-sdk/src/config.rs
deleted file mode 100644
index c4af5d6..0000000
--- a/crates/meta-feeder-sdk/src/config.rs
+++ /dev/null
@@ -1,312 +0,0 @@
-//! Self-describing per-plugin configuration: schema, redaction, merge, and the
-//! generic schema-driven HTML page the feeder serves at `/config`.
-//!
-//! The point of this module is that **the gateway carries no per-plugin config
-//! knowledge** and **the SDK carries no per-plugin field names**. A plugin
-//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema);
-//! everything here (redaction of secrets, merge-on-save, and the rendered form)
-//! is driven entirely by that schema. Add a feeder → it shows up with a working
-//! config form and zero UI/gateway changes.
-//!
-//! ## Persistence & precedence
-//!
-//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to
-//! `configure()`). The plugin reads **file-or-env** at startup: env is the
-//! first-boot seed and stays authoritative until the operator saves through the
-//! UI, at which point `config.json` exists and wins. There is no runtime reload
-//! — a save returns `restart_required: true` (mirrors the gateway's
-//! `gateway-config.json` posture).
-//!
-//! ## Secrets
-//!
-//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to
-//! the page (redacted to a `_set: bool`), and a blank incoming value on
-//! save means "keep the stored secret".
-
-use serde::{Deserialize, Serialize};
-use serde_json::{Map, Value};
-
-/// A plugin's full config surface — an ordered list of fields.
-#[derive(Serialize, Deserialize, Clone, Debug, Default)]
-pub struct ConfigSchema {
- pub fields: Vec,
-}
-
-impl ConfigSchema {
- /// Convenience: `true` when the plugin declares no configurable fields (the
- /// page then renders a "no configuration required" notice).
- pub fn is_empty(&self) -> bool {
- self.fields.is_empty()
- }
-}
-
-/// One configurable field. `record_array` fields nest `fields` for the
-/// per-record sub-schema (e.g. torznab's `indexers`).
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ConfigField {
- /// JSON key in the config object.
- pub key: String,
- /// Human label for the form.
- pub label: String,
- #[serde(rename = "type")]
- pub kind: FieldKind,
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub help: Option,
- #[serde(default)]
- pub required: bool,
- /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated
- /// as the record identifier for secret back-fill on merge (e.g. `name`).
- #[serde(default, skip_serializing_if = "Option::is_none")]
- pub fields: Option>,
-}
-
-impl ConfigField {
- pub fn text(key: &str, label: &str) -> Self {
- Self::new(key, label, FieldKind::Text)
- }
- pub fn secret(key: &str, label: &str) -> Self {
- Self::new(key, label, FieldKind::Secret)
- }
- pub fn bool(key: &str, label: &str) -> Self {
- Self::new(key, label, FieldKind::Bool)
- }
- pub fn number(key: &str, label: &str) -> Self {
- Self::new(key, label, FieldKind::Number)
- }
- pub fn list(key: &str, label: &str) -> Self {
- Self::new(key, label, FieldKind::List)
- }
- pub fn record_array(key: &str, label: &str, fields: Vec) -> Self {
- let mut f = Self::new(key, label, FieldKind::RecordArray);
- f.fields = Some(fields);
- f
- }
- fn new(key: &str, label: &str, kind: FieldKind) -> Self {
- Self {
- key: key.to_string(),
- label: label.to_string(),
- kind,
- help: None,
- required: false,
- fields: None,
- }
- }
- pub fn with_help(mut self, help: &str) -> Self {
- self.help = Some(help.to_string());
- self
- }
- pub fn required(mut self) -> Self {
- self.required = true;
- self
- }
-}
-
-/// Field widget / value type. Drives both rendering and the secret/merge rules.
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "snake_case")]
-pub enum FieldKind {
- /// Free text → JSON string.
- Text,
- /// Write-only secret → JSON string; redacted on read, "blank keeps" on save.
- Secret,
- /// Checkbox → JSON bool.
- Bool,
- /// Numeric input → JSON number.
- Number,
- /// Comma/line list → JSON array of strings.
- List,
- /// Repeatable record → JSON array of objects keyed by nested `fields`.
- RecordArray,
-}
-
-fn is_blank(v: Option<&Value>) -> bool {
- match v {
- None | Some(Value::Null) => true,
- Some(Value::String(s)) => s.is_empty(),
- _ => false,
- }
-}
-
-/// Produce a UI-safe view of `values`: secret fields are dropped and replaced
-/// with a `_set` boolean; everything else passes through. Recurses into
-/// `record_array` records for secret sub-fields.
-pub fn redact(values: &Value, schema: &ConfigSchema) -> Value {
- let src = values.as_object().cloned().unwrap_or_default();
- Value::Object(redact_object(&src, &schema.fields))
-}
-
-fn redact_object(src: &Map, fields: &[ConfigField]) -> Map {
- let mut out = Map::new();
- for f in fields {
- match f.kind {
- FieldKind::Secret => {
- let set = !is_blank(src.get(&f.key));
- out.insert(format!("{}_set", f.key), Value::Bool(set));
- }
- FieldKind::RecordArray => {
- let subs = f.fields.as_deref().unwrap_or(&[]);
- let arr = src
- .get(&f.key)
- .and_then(Value::as_array)
- .map(|recs| {
- recs.iter()
- .map(|r| {
- let m = r.as_object().cloned().unwrap_or_default();
- Value::Object(redact_object(&m, subs))
- })
- .collect::>()
- })
- .unwrap_or_default();
- out.insert(f.key.clone(), Value::Array(arr));
- }
- _ => {
- if let Some(v) = src.get(&f.key) {
- out.insert(f.key.clone(), v.clone());
- }
- }
- }
- }
- out
-}
-
-/// Merge `incoming` (from the UI) onto `base` (the stored/effective config),
-/// schema-aware. Returns the full, UNREDACTED object to persist:
-/// - secret: blank/absent incoming keeps `base`; non-blank overwrites.
-/// - text/number/bool/list: present in incoming overwrites; absent keeps base.
-/// - record_array: incoming array replaces base, but blank secret sub-fields are
-/// back-filled from the base record matched by the first sub-field (identifier).
-pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value {
- let base_obj = base.as_object().cloned().unwrap_or_default();
- let inc_obj = incoming.as_object().cloned().unwrap_or_default();
- Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields))
-}
-
-fn merge_object(
- base: &Map,
- inc: &Map,
- fields: &[ConfigField],
-) -> Map {
- let mut out = base.clone();
- for f in fields {
- match f.kind {
- FieldKind::Secret => {
- if !is_blank(inc.get(&f.key)) {
- out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap());
- }
- // blank/absent → keep base (no-op)
- }
- FieldKind::RecordArray => {
- let subs = f.fields.as_deref().unwrap_or(&[]);
- let id_key = subs.first().map(|s| s.key.clone());
- let base_recs = base.get(&f.key).and_then(Value::as_array).cloned();
- let inc_recs = match inc.get(&f.key).and_then(Value::as_array) {
- Some(r) => r.clone(),
- None => continue, // absent → keep base array untouched
- };
- let merged: Vec = inc_recs
- .iter()
- .map(|rec| {
- let rec_obj = rec.as_object().cloned().unwrap_or_default();
- // Find the matching base record by identifier sub-field.
- let base_match = match (&id_key, base_recs.as_ref()) {
- (Some(idk), Some(brs)) => brs
- .iter()
- .find(|br| br.get(idk) == rec_obj.get(idk))
- .and_then(Value::as_object)
- .cloned()
- .unwrap_or_default(),
- _ => Map::new(),
- };
- Value::Object(merge_object(&base_match, &rec_obj, subs))
- })
- .collect();
- out.insert(f.key.clone(), Value::Array(merged));
- }
- _ => {
- if let Some(v) = inc.get(&f.key) {
- out.insert(f.key.clone(), v.clone());
- }
- }
- }
- }
- out
-}
-
-/// The generic, schema-driven config page served at `GET /config`. Self-contained
-/// (inline CSS + JS, no external assets) so it loads identically when hit
-/// directly on the feeder OR reverse-proxied through the gateway. All API calls
-/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant.
-pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html");
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- fn torznab_like() -> ConfigSchema {
- ConfigSchema {
- fields: vec![
- ConfigField::secret("tmdb_token", "TMDB token"),
- ConfigField::bool("resolve_torrent_files", "Resolve .torrent"),
- ConfigField::record_array(
- "indexers",
- "Indexers",
- vec![
- ConfigField::text("name", "Name"),
- ConfigField::secret("api_key", "API key"),
- ],
- ),
- ],
- }
- }
-
- #[test]
- fn redact_hides_secrets_and_flags_set() {
- let v = json!({
- "tmdb_token": "abc",
- "resolve_torrent_files": true,
- "indexers": [{"name":"prowlarr","api_key":"k"}]
- });
- let r = redact(&v, &torznab_like());
- assert_eq!(r["tmdb_token_set"], json!(true));
- assert!(r.get("tmdb_token").is_none());
- assert_eq!(r["resolve_torrent_files"], json!(true));
- assert_eq!(r["indexers"][0]["name"], json!("prowlarr"));
- assert_eq!(r["indexers"][0]["api_key_set"], json!(true));
- assert!(r["indexers"][0].get("api_key").is_none());
- }
-
- #[test]
- fn merge_blank_secret_keeps_base() {
- let base = json!({"tmdb_token": "keepme"});
- let inc = json!({"tmdb_token": ""});
- let m = merge(&base, &inc, &torznab_like());
- assert_eq!(m["tmdb_token"], json!("keepme"));
- }
-
- #[test]
- fn merge_nonblank_secret_overwrites() {
- let base = json!({"tmdb_token": "old"});
- let inc = json!({"tmdb_token": "new"});
- let m = merge(&base, &inc, &torznab_like());
- assert_eq!(m["tmdb_token"], json!("new"));
- }
-
- #[test]
- fn merge_record_array_backfills_blank_secret_by_identifier() {
- let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]});
- // UI re-submits the indexer with a blank api_key → keep the stored one.
- let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]});
- let m = merge(&base, &inc, &torznab_like());
- assert_eq!(m["indexers"][0]["api_key"], json!("secret"));
- assert_eq!(m["indexers"][0]["name"], json!("prowlarr"));
- }
-
- #[test]
- fn merge_bool_overwrites_when_present() {
- let base = json!({"resolve_torrent_files": true});
- let inc = json!({"resolve_torrent_files": false});
- let m = merge(&base, &inc, &torznab_like());
- assert_eq!(m["resolve_torrent_files"], json!(false));
- }
-}
diff --git a/crates/meta-feeder-sdk/src/config_page.html b/crates/meta-feeder-sdk/src/config_page.html
deleted file mode 100644
index 17d9d74..0000000
--- a/crates/meta-feeder-sdk/src/config_page.html
+++ /dev/null
@@ -1,268 +0,0 @@
-
-
-
-
-
-feeder config
-
-
-
-
Plugin configuration
-
Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked * are required.
-
-
-
-
-
-
-
-
-
diff --git a/crates/meta-feeder-sdk/src/domain.rs b/crates/meta-feeder-sdk/src/domain.rs
deleted file mode 100644
index 0e90b5e..0000000
--- a/crates/meta-feeder-sdk/src/domain.rs
+++ /dev/null
@@ -1,549 +0,0 @@
-//! `contentKind` → `domain` / `workForm` — the routing tables from
-//! METADATA_KEYS.md §1.
-//!
-//! `domain` answers **"which application is this record destined for?"**. It
-//! is the coarsest facet in the registry and the key every client filters its
-//! catalogue on — **one value per app**, no alternation:
-//!
-//! ```text
-//! meta-watch → domain:screen
-//! meta-listen → domain:music
-//! meta-read → domain:literature
-//! ```
-//!
-//! `workForm` answers the *other* question the old `film`/`tv` split was
-//! secretly encoding: **is this work complete in itself, or one instalment of
-//! a run?** (`standalone` | `serial`). It is cross-domain — a comic issue is
-//! `serial` in `literature` exactly as an episode is `serial` in `screen` —
-//! which is what lets a client ask for it independently of the wall.
-//!
-//! # Why they are stored and not derived on read
-//!
-//! `contentKind`'s vocabulary is deliberately open — a new feeder may mint a
-//! kind and ship without a fleet-wide update. That only stays safe because the
-//! domain travels *with* the record: a peer that has never heard of
-//! `podcastEpisode` still routes it correctly, because it never has to
-//! interpret the token. Deriving on read would silently drop every record
-//! whose kind the reader predates. The same argument applies to `workForm`,
-//! plus a second one: the two kinds that most need it (`pack` and `card`) are
-//! exactly the two the kind alone cannot decide.
-//!
-//! # ⚠ The binding is permanent and one-way
-//!
-//! Add kinds freely; **never re-map an existing kind to a different domain**.
-//! Because the value is stored, two peers on different versions that disagree
-//! about where `audiobook` belongs produce a split-brain that redeploying
-//! cannot heal — the wrong domain is already written into records across the
-//! mesh. If a kind genuinely has to move, mint a new kind.
-//!
-//! The `film`+`tv` → `screen` merge (METADATA_KEYS.md §14.17) is the one
-//! deliberate exception, taken once and early and paid for with a one-shot
-//! meta-core corpus sweep. It is not a precedent.
-//!
-//! # Why `domain` and not one kind-key per medium
-//!
-//! `record_matches` is AND-across-keys / OR-within-key, and a non-matching key
-//! fails the record outright. A per-medium split (`videoKind` / `audioKind` /
-//! `documentKind`) would force meta-listen to express "music video **or**
-//! audio track" as an OR across two *different* keys — which the query
-//! language cannot represent at all. One domain key, OR-ed within itself,
-//! always can. `workForm` is one key everywhere for the same reason.
-
-/// The `contentKind` → `domain` table. A kind absent from this list has no
-/// domain and is **not independently routable** — correct for sidecars
-/// (subtitles, artwork) and for un-anchored hits, a producer bug for anything
-/// else.
-///
-/// ⚠ `pack` is deliberately absent. A season pack and an album release are the
-/// same structural thing, so the kind alone cannot tell `screen` from `music`;
-/// the writer stamps `pack`'s domain from its own context (the same reason
-/// `derive_file_type` cannot have a second `pack` arm).
-///
-/// ⚠ `podcast` / `podcastEpisode` are absent too: the `spoken` domain was
-/// retired unused (METADATA_KEYS.md §14.17), and the PR that first *writes*
-/// those kinds picks their domain.
-const DOMAIN_BY_CONTENT_KIND: &[(&str, &str)] = &[
- // screen — everything meta-watch consumes, films and serials alike. The
- // standalone/serial split lives in `workForm`, not here.
- ("movie", "screen"),
- ("series", "screen"),
- ("episode", "screen"),
- // music
- ("track", "music"),
- ("album", "music"),
- ("artist", "music"),
- ("musicVideo", "music"),
- ("djMix", "music"),
- ("liveSet", "music"),
- // literature — an audiobook's identity is the book (same author, same ISBN
- // family, same series as the ebook), so meta-read groups the two editions
- // on one page. The player does not decide the domain.
- ("book", "literature"),
- ("audiobook", "literature"),
- ("comic", "literature"),
- ("manga", "literature"),
- ("magazine", "literature"),
- // science
- ("paper", "science"),
-];
-
-/// The `contentKind` → `workForm` table (METADATA_KEYS.md §1).
-///
-/// `standalone` = a work complete in itself. `serial` = an instalment of an
-/// ongoing run, **or the container or headline for one** — which is why a
-/// `series` card and an `episode` are both `serial`.
-///
-/// ⚠ `pack` is absent for the same reason it is absent above: a season pack is
-/// `serial`, an album release is `standalone`, and only the writer knows which.
-const WORK_FORM_BY_CONTENT_KIND: &[(&str, &str)] = &[
- // screen
- ("movie", "standalone"),
- ("series", "serial"),
- ("episode", "serial"),
- // music — an album is a closed work, not an ongoing run, so every rung of
- // the music ladder is standalone.
- ("track", "standalone"),
- ("album", "standalone"),
- ("artist", "standalone"),
- ("musicVideo", "standalone"),
- ("djMix", "standalone"),
- ("liveSet", "standalone"),
- // literature — the split that proves `workForm` is not a screen-only axis.
- ("book", "standalone"),
- ("audiobook", "standalone"),
- ("comic", "serial"),
- ("manga", "serial"),
- ("magazine", "serial"),
- // science
- ("paper", "standalone"),
-];
-
-/// Resolve the domain a `contentKind` belongs to.
-///
-/// Returns `None` for `pack` (not derivable from the kind — the writer supplies
-/// it), for format/sidecar kinds that route nowhere, and for any kind this
-/// build has never heard of. A caller that already knows the domain from its
-/// own context must prefer that over this table.
-pub fn domain_for_content_kind(content_kind: &str) -> Option<&'static str> {
- lookup(DOMAIN_BY_CONTENT_KIND, content_kind)
-}
-
-/// Resolve the work form a `contentKind` sits on. Same contract as
-/// [`domain_for_content_kind`]: `None` means "not derivable", never a guess.
-pub fn work_form_for_content_kind(content_kind: &str) -> Option<&'static str> {
- lookup(WORK_FORM_BY_CONTENT_KIND, content_kind)
-}
-
-fn lookup(table: &'static [(&'static str, &'static str)], key: &str) -> Option<&'static str> {
- let k = key.trim();
- table
- .iter()
- .find(|(kind, _)| kind.eq_ignore_ascii_case(k))
- .map(|(_, value)| *value)
-}
-
-/// Kinds whose `domain` the kind→domain table deliberately cannot decide, so
-/// the **writer** supplies it (METADATA_KEYS.md §1). Today that is `pack`
-/// alone: a season pack and an album release are the same structural thing, and
-/// only the producer knows which — indexer-feeder's `bt.rs` reads the domain off
-/// the per-file kind *before* overwriting it with `pack` precisely so the value
-/// survives.
-///
-/// They must ride along in every `domain:` → `contentKind:` expansion. The
-/// expansion is only ever a **pre-filter** — the real `domain:` term is enforced
-/// afterwards by [`crate::query_eval::record_matches`] — so widening it is free,
-/// while omitting these kinds would drop exactly the records whose `domain` is
-/// the most authoritative one on the wire.
-pub const WRITER_SUPPLIED_DOMAIN_KINDS: &[&str] = &["pack"];
-
-/// The `workForm` twin of [`WRITER_SUPPLIED_DOMAIN_KINDS`], and for the same
-/// reason: a season pack is `serial`, an album release is `standalone`.
-pub const WRITER_SUPPLIED_WORK_FORM_KINDS: &[&str] = &["pack"];
-
-/// The inverse: every `contentKind` that belongs to `domain`.
-///
-/// Used at the gateway-routing boundary. A gateway declares the *content kinds*
-/// it serves (`served_content_kinds`), not domains, so a `domain:` filter is
-/// matched by expanding it here rather than by adding a `served_domains` field
-/// to the heartbeat — which would be a wire-protocol change requiring every
-/// feeder manifest and every peer to move at once.
-///
-/// ⚠ Since the `film`+`tv` merge this returns **several** kinds for `screen`
-/// (`movie`, `series`, `episode`) where the two old values were disjoint. A
-/// caller that reads only the first element silently means "movie" — scan the
-/// whole list, or narrow with a `workForm:` term first.
-pub fn content_kinds_for_domain(domain: &str) -> Vec<&'static str> {
- inverse(DOMAIN_BY_CONTENT_KIND, domain)
-}
-
-/// The inverse of the `workForm` table. Same routing-boundary role, same
-/// scan-the-whole-list warning as [`content_kinds_for_domain`].
-pub fn content_kinds_for_work_form(work_form: &str) -> Vec<&'static str> {
- inverse(WORK_FORM_BY_CONTENT_KIND, work_form)
-}
-
-fn inverse(table: &'static [(&'static str, &'static str)], value: &str) -> Vec<&'static str> {
- let v = value.trim();
- table
- .iter()
- .filter(|(_, val)| val.eq_ignore_ascii_case(v))
- .map(|(kind, _)| *kind)
- .collect()
-}
-
-/// Rewrite a query so a `domain:` / `workForm:` filter can reach a plugin that
-/// has never heard of either key.
-///
-/// Returns `None` when the query carries neither — the common path, and cheaper
-/// than cloning.
-///
-/// # Why translate instead of teaching every plugin
-///
-/// `record_matches` fails a record outright when a filter key is missing from
-/// its fields, and `domain` is missing from every record a not-yet-updated
-/// feeder emits (`workForm` from every record written before it existed).
-/// Most feeders pin the SDK by git tag, so they cannot pick up a new key on our
-/// schedule. Translating `domain:literature` into the `contentKind:`
-/// alternation it covers means the wire query a plugin sees never mentions
-/// either key — old plugins keep working unchanged, while the dispatcher still
-/// enforces the real filters afterwards, against records it has just stamped.
-///
-/// # The two axes intersect
-///
-/// `domain:screen workForm:serial` covers `series` and `episode` — the
-/// **intersection**, not the union. Widening to the union would hand a serial
-/// query the whole film corpus as a pre-filter; the post-stamp check would then
-/// throw it away, having already paid for it upstream.
-///
-/// An explicit `contentKind:` filter already in the query is left alone: it is
-/// the narrower of the three, and the `domain`/`workForm` terms are still
-/// enforced post-stamp, so the AND semantics hold either way.
-pub fn expand_routing_filters(
- query: &crate::query::GatewayQuery,
-) -> Option {
- let domains = query.filters.get("domain");
- let work_forms = query.filters.get("workForm");
- if domains.is_none() && work_forms.is_none() {
- return None;
- }
-
- let by_domain = domains.map(|vs| expand_values(vs, content_kinds_for_domain));
- let by_work_form = work_forms.map(|vs| expand_values(vs, content_kinds_for_work_form));
-
- let mut out = query.clone();
- out.filters.remove("domain");
- out.filters.remove("workForm");
- if out.filters.contains_key("contentKind") {
- return Some(out);
- }
-
- let mut kinds: Vec<&'static str> = match (by_domain, by_work_form) {
- (Some(d), Some(w)) => d.into_iter().filter(|k| w.contains(k)).collect(),
- (Some(d), None) => d,
- (None, Some(w)) => w,
- (None, None) => unreachable!("guarded above"),
- };
- // Domain-ambiguous kinds come along whenever the expansion is non-empty
- // — see [`WRITER_SUPPLIED_DOMAIN_KINDS`]. Skipped when nothing mapped,
- // so an unknown value still expands to nothing and leaves the filter
- // out entirely (below) rather than narrowing to `contentKind:pack`.
- if !kinds.is_empty() {
- for k in WRITER_SUPPLIED_DOMAIN_KINDS
- .iter()
- .chain(WRITER_SUPPLIED_WORK_FORM_KINDS.iter())
- {
- if !kinds.contains(k) {
- kinds.push(k);
- }
- }
- }
- // An unknown value — or two axes with no kind in common — expands to
- // nothing. Inserting an empty alternation would drop every record at the
- // plugin; leaving the filter out lets the plugin answer and the post-stamp
- // check do the (correct, empty) filtering.
- if !kinds.is_empty() {
- out.filters.insert(
- "contentKind".to_string(),
- kinds.into_iter().map(|k| k.to_string()).collect(),
- );
- }
- Some(out)
-}
-
-/// Deprecated alias for [`expand_routing_filters`], kept so a caller pinned to
-/// an older SDK keeps compiling. It expands `workForm:` too — the name is the
-/// only thing that is stale.
-#[deprecated(note = "renamed to expand_routing_filters; it now expands workForm: as well")]
-pub fn expand_domain_filter(
- query: &crate::query::GatewayQuery,
-) -> Option {
- expand_routing_filters(query)
-}
-
-fn expand_values(
- values: &[String],
- expand: fn(&str) -> Vec<&'static str>,
-) -> Vec<&'static str> {
- let mut out: Vec<&'static str> = Vec::new();
- for v in values {
- for k in expand(v) {
- if !out.contains(&k) {
- out.push(k);
- }
- }
- }
- out
-}
-
-/// Stamp `domain` on a field map that already carries a `contentKind`.
-///
-/// The convention from METADATA_KEYS.md §1: **whoever writes `contentKind`
-/// writes `domain` in the same breath**. An existing `domain` is never
-/// overwritten — a writer that knows better than the table (a `pack`, say) has
-/// already set it.
-pub fn stamp_domain(fields: &mut std::collections::BTreeMap) {
- stamp(fields, "domain", domain_for_content_kind);
-}
-
-/// Stamp `workForm` the same way, from the same `contentKind`. Call it
-/// wherever [`stamp_domain`] is called — the registry treats the two as one
-/// breath.
-pub fn stamp_work_form(fields: &mut std::collections::BTreeMap) {
- stamp(fields, "workForm", work_form_for_content_kind);
-}
-
-fn stamp(
- fields: &mut std::collections::BTreeMap,
- key: &str,
- derive: fn(&str) -> Option<&'static str>,
-) {
- if fields.contains_key(key) {
- return;
- }
- let Some(kind) = fields.get("contentKind").cloned() else {
- return;
- };
- if let Some(value) = derive(&kind) {
- fields.insert(key.to_string(), value.to_string());
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use std::collections::BTreeMap;
-
- #[test]
- fn kinds_map_to_their_domain() {
- assert_eq!(domain_for_content_kind("movie"), Some("screen"));
- assert_eq!(domain_for_content_kind("episode"), Some("screen"));
- assert_eq!(domain_for_content_kind("track"), Some("music"));
- // The case that proves medium and domain are different axes.
- assert_eq!(domain_for_content_kind("musicVideo"), Some("music"));
- // The case that proves the player does not decide the domain.
- assert_eq!(domain_for_content_kind("audiobook"), Some("literature"));
- }
-
- #[test]
- fn film_and_tv_are_one_domain_now() {
- // The merge (METADATA_KEYS.md §14.17): meta-watch's wall is one term.
- assert_eq!(
- domain_for_content_kind("movie"),
- domain_for_content_kind("series")
- );
- // ...and the split it used to carry lives here instead.
- assert_eq!(work_form_for_content_kind("movie"), Some("standalone"));
- assert_eq!(work_form_for_content_kind("series"), Some("serial"));
- assert_eq!(work_form_for_content_kind("episode"), Some("serial"));
- }
-
- #[test]
- fn work_form_is_not_a_screen_only_axis() {
- // The point of the key: it means the same thing in every domain.
- assert_eq!(work_form_for_content_kind("comic"), Some("serial"));
- assert_eq!(work_form_for_content_kind("book"), Some("standalone"));
- assert_eq!(work_form_for_content_kind("album"), Some("standalone"));
- }
-
- #[test]
- fn the_spoken_domain_is_gone() {
- // Retired unused. The kinds stay reserved with no domain until the PR
- // that writes them picks one.
- assert_eq!(domain_for_content_kind("podcast"), None);
- assert_eq!(domain_for_content_kind("podcastEpisode"), None);
- assert!(content_kinds_for_domain("spoken").is_empty());
- }
-
- #[test]
- fn pack_has_no_derivable_domain_or_work_form() {
- assert_eq!(domain_for_content_kind("pack"), None);
- assert_eq!(work_form_for_content_kind("pack"), None);
- }
-
- #[test]
- fn unknown_kind_is_none_not_a_guess() {
- assert_eq!(domain_for_content_kind("hologram"), None);
- assert_eq!(work_form_for_content_kind("hologram"), None);
- }
-
- #[test]
- fn expansion_is_the_inverse() {
- let music = content_kinds_for_domain("music");
- assert!(music.contains(&"track"));
- assert!(music.contains(&"musicVideo"));
- assert!(!music.contains(&"audiobook"));
- for kind in content_kinds_for_domain("literature") {
- assert_eq!(domain_for_content_kind(kind), Some("literature"));
- }
- for kind in content_kinds_for_work_form("serial") {
- assert_eq!(work_form_for_content_kind(kind), Some("serial"));
- }
- }
-
- #[test]
- fn screen_expands_to_every_rung_not_just_the_first() {
- // The regression the merge creates for any caller reading `.first()`.
- let screen = content_kinds_for_domain("screen");
- assert!(screen.contains(&"movie"));
- assert!(screen.contains(&"series"));
- assert!(screen.contains(&"episode"));
- }
-
- fn q(filters: &[(&str, &[&str])]) -> crate::query::GatewayQuery {
- crate::query::GatewayQuery {
- raw_text: String::new(),
- free_text: String::new(),
- filters: filters
- .iter()
- .map(|(k, vs)| {
- (
- k.to_string(),
- vs.iter().map(|v| v.to_string()).collect::>(),
- )
- })
- .collect(),
- ranges: Vec::new(),
- negations: Vec::new(),
- }
- }
-
- #[test]
- fn no_routing_filter_is_left_untouched() {
- assert!(expand_routing_filters(&q(&[("fileType", &["video"])])).is_none());
- }
-
- #[test]
- fn domain_becomes_the_content_kinds_it_covers() {
- let out = expand_routing_filters(&q(&[("domain", &["literature"])])).unwrap();
- // The plugin never sees `domain:` — that is the whole point.
- assert!(!out.filters.contains_key("domain"));
- let kinds = out.filters.get("contentKind").unwrap();
- assert!(kinds.contains(&"book".to_string()));
- assert!(kinds.contains(&"audiobook".to_string()));
- assert!(!kinds.contains(&"track".to_string()));
- }
-
- #[test]
- fn work_form_alone_also_expands() {
- let out = expand_routing_filters(&q(&[("workForm", &["serial"])])).unwrap();
- assert!(!out.filters.contains_key("workForm"));
- let kinds = out.filters.get("contentKind").unwrap();
- assert!(kinds.contains(&"episode".to_string()));
- assert!(kinds.contains(&"comic".to_string()));
- assert!(!kinds.contains(&"movie".to_string()));
- }
-
- #[test]
- fn the_two_axes_intersect_they_do_not_union() {
- let out = expand_routing_filters(&q(&[
- ("domain", &["screen"][..]),
- ("workForm", &["serial"][..]),
- ]))
- .unwrap();
- let kinds = out.filters.get("contentKind").unwrap();
- assert!(kinds.contains(&"series".to_string()));
- assert!(kinds.contains(&"episode".to_string()));
- // The union would have dragged the whole film corpus in.
- assert!(!kinds.contains(&"movie".to_string()));
- // A `serial` comic is in `literature`, not on meta-watch's wall.
- assert!(!kinds.contains(&"comic".to_string()));
- }
-
- #[test]
- fn an_explicit_content_kind_filter_wins() {
- let out = expand_routing_filters(&q(&[
- ("domain", &["music"][..]),
- ("contentKind", &["track"][..]),
- ]))
- .unwrap();
- assert_eq!(
- out.filters.get("contentKind").map(Vec::as_slice),
- Some(&["track".to_string()][..])
- );
- }
-
- #[test]
- fn expansion_carries_the_writer_supplied_kinds() {
- // `pack` has no domain in the table (a season pack and an album are the
- // same structural thing), so the writer stamps one. Leaving it out of
- // the alternation would make a `domain:screen` pre-filter drop the whole
- // torrent-pack corpus, which `record_matches` would then never see.
- let out = expand_routing_filters(&q(&[("domain", &["screen"])])).unwrap();
- let kinds = out.filters.get("contentKind").unwrap();
- assert!(kinds.contains(&"episode".to_string()));
- assert!(kinds.contains(&"pack".to_string()));
- // ...and it rides along on the `workForm` axis too, exactly once.
- let out = expand_routing_filters(&q(&[("workForm", &["serial"])])).unwrap();
- let kinds = out.filters.get("contentKind").unwrap();
- assert_eq!(kinds.iter().filter(|k| *k == "pack").count(), 1);
- }
-
- #[test]
- fn unknown_value_adds_no_alternation() {
- // An empty alternation would drop every record at the plugin; the
- // post-stamp check does the (correct, empty) filtering.
- let out = expand_routing_filters(&q(&[("domain", &["hologram"])])).unwrap();
- // Not even `pack`: a lone `contentKind:pack` would be a NARROWING of a
- // filter that should widen to nothing.
- assert!(!out.filters.contains_key("contentKind"));
- }
-
- #[test]
- fn two_axes_with_nothing_in_common_widen_rather_than_narrow() {
- // No music kind is `serial`. The pre-filter drops out; the post-stamp
- // check still enforces both terms.
- let out = expand_routing_filters(&q(&[
- ("domain", &["music"][..]),
- ("workForm", &["serial"][..]),
- ]))
- .unwrap();
- assert!(!out.filters.contains_key("contentKind"));
- }
-
- #[test]
- fn stamp_fills_in_and_never_overwrites() {
- let mut f: BTreeMap = BTreeMap::new();
- f.insert("contentKind".into(), "episode".into());
- stamp_domain(&mut f);
- stamp_work_form(&mut f);
- assert_eq!(f.get("domain").map(String::as_str), Some("screen"));
- assert_eq!(f.get("workForm").map(String::as_str), Some("serial"));
-
- // A pack writer that already knows its context keeps it.
- let mut p: BTreeMap = BTreeMap::new();
- p.insert("contentKind".into(), "pack".into());
- p.insert("domain".into(), "music".into());
- p.insert("workForm".into(), "standalone".into());
- stamp_domain(&mut p);
- stamp_work_form(&mut p);
- assert_eq!(p.get("domain").map(String::as_str), Some("music"));
- assert_eq!(p.get("workForm").map(String::as_str), Some("standalone"));
-
- // No contentKind → nothing invented.
- let mut n: BTreeMap = BTreeMap::new();
- stamp_domain(&mut n);
- stamp_work_form(&mut n);
- assert!(n.get("domain").is_none());
- assert!(n.get("workForm").is_none());
- }
-}
diff --git a/crates/meta-feeder-sdk/src/enrich.rs b/crates/meta-feeder-sdk/src/enrich.rs
deleted file mode 100644
index b55a9f8..0000000
--- a/crates/meta-feeder-sdk/src/enrich.rs
+++ /dev/null
@@ -1,614 +0,0 @@
-//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline.
-//!
-//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base
-//! record, then drives the **reused meta-sort enrichment plugins**
-//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each
-//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to
-//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches
-//! enrichment — full separation of concerns.
-//!
-//! ## Ordering — why we poll
-//!
-//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from
-//! the record), and those are produced by filename-parser. So the pipeline is
-//! **filename-parser → tmdb**. The plugin `/process` contract is *async*
-//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after
-//! kicking filename-parser we **poll meta-core** until `originalTitle` appears
-//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps
-//! the feeder stateless — no callback-correlation bookkeeping. The plugins'
-//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`.
-//!
-//! The whole pipeline runs in a spawned background task so it never adds latency
-//! to the libp2p resolve path; the meta-share ingester propagates the enriched
-//! record on its next poll (~eventually consistent).
-
-use std::collections::BTreeMap;
-use std::sync::Arc;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-use serde::Serialize;
-use tracing::{debug, warn};
-
-use crate::meta_core::{self, Provenance};
-use crate::types::DiscoveryRecord;
-
-/// Operator-supplied enrichment wiring, parsed once from the feeder env.
-#[derive(Clone, Debug)]
-pub struct EnrichmentConfig {
- /// meta-core root the feeder self-publishes to + hands to the plugins.
- pub meta_core_url: String,
- /// `metamesh-plugin-filename-parser` base URL (optional).
- pub filename_parser_url: Option,
- /// `metamesh-plugin-tmdb` base URL (optional).
- pub tmdb_url: Option,
- /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`.
- pub tmdb_token: Option,
- /// TMDB metadata language (default `en-US`).
- pub tmdb_language: String,
- /// Stable per-deployment id stamped into `provenance.gatewayPeer`.
- pub gateway_peer: String,
- /// URL the plugins POST their completion callback to (the feeder's own
- /// no-op `/enrich/callback`). Polling is the real completion signal; this
- /// only keeps the plugins from logging a failed-callback warning.
- pub callback_url: String,
- /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the
- /// pipeline drives subtitle fetching after tmdb resolves the ids — the
- /// single reference implementation of OpenSubtitles fetching (the gateway
- /// no longer carries its own).
- pub opensubtitles_url: Option,
- /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`.
- pub opensubtitles_api_key: Option,
- /// OpenSubtitles account username (required by the plugin for downloads).
- pub opensubtitles_username: Option,
- /// OpenSubtitles account password.
- pub opensubtitles_password: Option,
- /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
- pub opensubtitles_languages: String,
-}
-
-fn env_nonempty(key: &str) -> Option {
- std::env::var(key)
- .ok()
- .map(|s| s.trim().to_string())
- .filter(|s| !s.is_empty())
-}
-
-/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the
-/// app-compose convention `-feeder` → `-:8080`
-/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets
-/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit
-/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns
-/// `None` when `HOSTNAME` is unset or not in `-feeder` form, in which case
-/// the caller falls back to the env seed (then soft-skips that plugin).
-fn derive_sidecar_url(plugin: &str) -> Option {
- let host = env_nonempty("HOSTNAME")?;
- let prefix = host.strip_suffix("-feeder")?;
- Some(format!("http://{prefix}-{plugin}:8080"))
-}
-
-impl EnrichmentConfig {
- /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset —
- /// the **bundled-feeder** path (records returned over HTTP, the gateway
- /// dispatcher stores them). Self-publishing feeders that source meta-core from
- /// their own dashboard config use [`from_meta_core`](Self::from_meta_core).
- pub fn from_env() -> Option {
- Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?))
- }
-
- /// Build with an explicit meta-core root — sourced from the feeder's own
- /// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The
- /// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the
- /// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest
- /// keep their env seeds / defaults. Secrets (tmdb token, language) are
- /// overlaid by the caller from the per-feeder config.
- pub fn from_meta_core(meta_core_url: String) -> Self {
- EnrichmentConfig {
- meta_core_url,
- filename_parser_url: env_nonempty("FILENAME_PARSER_URL")
- .or_else(|| derive_sidecar_url("filename-parser")),
- tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")),
- tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")),
- tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()),
- gateway_peer: env_nonempty("META_GATEWAY_PEER_ID")
- .or_else(|| env_nonempty("HOSTNAME"))
- .unwrap_or_else(|| "gateway-feeder".to_string()),
- callback_url: env_nonempty("META_FEEDER_CALLBACK_URL")
- .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()),
- opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"),
- opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"),
- opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"),
- opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
- opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
- .unwrap_or_else(|| "en".to_string()),
- }
- }
-}
-
-/// How long to wait for filename-parser to merge `originalTitle` before handing
-/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous
-/// ceiling, not an expected wait.
-const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5);
-/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the
-/// sibling files. tmdb hits the network, so a longer ceiling than filename.
-const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20);
-const POLL_INTERVAL: Duration = Duration::from_millis(200);
-
-/// Show-level TMDB fields that are **fanned** from the primary file to every
-/// sibling file in a multi-file (season-pack) torrent — they're identical for
-/// every episode of a show, so we resolve tmdb **once** and copy these across
-/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which
-/// gateway BT records don't carry, so without this each file would re-hit TMDB).
-/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`,
-/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file.
-const ANCHOR_EXACT: &[&str] = &[
- "tmdbid",
- "imdbid",
- "poster",
- "backdrop",
- "rating",
- "releasedate",
- "studio",
- "genres",
- "tags",
-];
-const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"];
-
-/// One file (or pack) to enrich: its CID, the filename to parse, and the base
-/// fields already written to meta-core (handed to the plugins as `existingMeta`).
-#[derive(Clone, Debug)]
-pub struct EnrichTarget {
- pub cid: String,
- pub file_name: String,
- pub base: BTreeMap,
-}
-
-/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to
-/// clone (just a reqwest client + an `Arc`), so it can be moved into a
-/// spawned per-resolve task.
-#[derive(Clone)]
-pub struct Enricher {
- http: reqwest::Client,
- cfg: Arc,
-}
-
-#[derive(Serialize)]
-struct ProcessRequest<'a> {
- #[serde(rename = "taskId")]
- task_id: String,
- cid: &'a str,
- #[serde(rename = "filePath")]
- file_path: &'a str,
- #[serde(rename = "callbackUrl")]
- callback_url: &'a str,
- #[serde(rename = "metaCoreUrl")]
- meta_core_url: &'a str,
- #[serde(rename = "existingMeta")]
- existing_meta: &'a BTreeMap,
-}
-
-impl Enricher {
- pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self {
- Enricher {
- http,
- cfg: Arc::new(cfg),
- }
- }
-
- pub fn meta_core_url(&self) -> &str {
- &self.cfg.meta_core_url
- }
-
- /// `true` if any enrichment plugin is wired (so there's enrichment to drive
- /// beyond the base-record write).
- pub fn has_plugins(&self) -> bool {
- self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some()
- }
-
- /// Write the feeder's base record into meta-core (merge/upsert) under `cid`.
- /// Identical shape to the gateway dispatcher's write (provenance + the
- /// `cids/` key-set collapse).
- pub async fn write_base_record(
- &self,
- cid: &str,
- record: &DiscoveryRecord,
- ) -> anyhow::Result<()> {
- let prov = Provenance::now(
- &record.upstream_id,
- &record.record_id,
- &self.cfg.gateway_peer,
- );
- let body = meta_core::build_metadata_body(record, &prov);
- meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await
- }
-
- /// Run the video enrichment pipeline for one resolved CID: filename-parser
- /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each
- /// plugin writes its own fields to meta-core. Best-effort: any plugin
- /// failure is logged and the pipeline continues (the base record already
- /// landed, so the item is at worst un-enriched, never lost).
- pub async fn enrich_video(
- &self,
- cid: &str,
- file_name: &str,
- base_fields: BTreeMap,
- ) {
- let mut meta = base_fields;
-
- // 1. filename-parser → originalTitle / movieYear / videoType / season / episode.
- if let Some(fp_url) = self.cfg.filename_parser_url.clone() {
- match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await {
- Ok(()) => {
- // Poll for the merged fields so tmdb gets the clean title.
- if let Some(merged) = self
- .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
- .await
- {
- meta = merged;
- } else {
- debug!(
- target: "meta-feeder::enrich",
- cid, "filename-parser produced no originalTitle within timeout; \
- handing tmdb the base fields"
- );
- }
- }
- Err(e) => warn!(
- target: "meta-feeder::enrich",
- cid, error = %e, "filename-parser /process failed (non-fatal)"
- ),
- }
- }
-
- // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video +
- // a clean originalTitle, both now present on `meta`).
- if let Some(tmdb_url) = self.cfg.tmdb_url.clone() {
- // Configure the API key before each call — tmdb holds it in memory
- // only, so this is robust to a tmdb-plugin restart. Cheap (local).
- if let Err(e) = self.configure_tmdb(&tmdb_url).await {
- warn!(
- target: "meta-feeder::enrich",
- cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)"
- );
- }
- if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await {
- warn!(
- target: "meta-feeder::enrich",
- cid, error = %e, "tmdb /process failed (non-fatal)"
- );
- }
- }
-
- // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which
- // tmdb writes async). Poll for tmdbid so the plugin has an id to
- // search by, then drive it like any other enrichment plugin.
- if self.cfg.opensubtitles_url.is_some() {
- let merged = self
- .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT)
- .await
- .unwrap_or(meta);
- self.enrich_subtitles(cid, file_name, &merged).await;
- }
- }
-
- /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files +
- /// an optional whole-torrent pack record. filename-parser runs on every
- /// target; tmdb runs **once** on the primary video file and its show-level
- /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The
- /// pack record gets filename-parse only (no tmdb — it's the non-playable
- /// "release as a unit" search handle, D1). Best-effort throughout.
- pub async fn enrich_bundle(&self, videos: Vec, pack: Option) {
- // Pack: filename-parse only, no tmdb.
- if let Some(p) = pack {
- self.fire_filename(&p).await;
- }
-
- if videos.is_empty() {
- return;
- }
- if videos.len() == 1 {
- // Degenerate single-video bundle → the plain pipeline (filename → tmdb).
- let v = &videos[0];
- self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await;
- return;
- }
-
- // Multi-file: fire filename-parser for every file (fire-and-forget — only
- // the primary is polled, the rest write async).
- for v in &videos {
- self.fire_filename(v).await;
- }
-
- // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen).
- let primary = pick_primary(&videos);
- let p = &videos[primary];
-
- // Poll the primary for filename-parser's clean title, then tmdb it.
- let merged = self
- .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
- .await
- .unwrap_or_else(|| p.base.clone());
-
- let Some(tmdb_url) = self.cfg.tmdb_url.clone() else {
- return; // no tmdb wired → filename-only enrichment, done.
- };
- if let Err(e) = self.configure_tmdb(&tmdb_url).await {
- warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed");
- }
- if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await {
- warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed");
- return;
- }
-
- // Read the resolved anchor back off the primary, then fan it to siblings.
- let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await
- else {
- debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan");
- return;
- };
- let anchor = extract_anchor(&anchor_rec);
- if anchor.is_empty() {
- return;
- }
- for (i, v) in videos.iter().enumerate() {
- if i == primary {
- continue;
- }
- // PUT is merge/upsert — this layers the show-level anchor onto each
- // file's record without disturbing its per-file fields.
- if let Err(e) =
- meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await
- {
- warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
- }
- }
-
- // Subtitles per video: each episode needs its own (its season/episode
- // differs), but they share the fanned tmdbid. Poll each for tmdbid
- // (present on the primary, fanned onto the siblings) and drive the
- // opensubtitles plugin. Best-effort + quota-guarded inside the plugin.
- if self.cfg.opensubtitles_url.is_some() {
- for v in &videos {
- let merged = self
- .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT)
- .await
- .unwrap_or_else(|| v.base.clone());
- self.enrich_subtitles(&v.cid, &v.file_name, &merged).await;
- }
- }
- }
-
- /// Fire filename-parser `/process` for one target (fire-and-forget; the
- /// plugin merges its fields to meta-core async).
- async fn fire_filename(&self, t: &EnrichTarget) {
- if let Some(fp_url) = self.cfg.filename_parser_url.clone() {
- if let Err(e) = self
- .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base)
- .await
- {
- warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed");
- }
- }
- }
-
- /// POST `/process` to a plugin. Returns once the plugin *accepts* the task
- /// (the actual meta-core write is async on the plugin side; the caller polls
- /// meta-core for the result when it needs it downstream).
- async fn process(
- &self,
- plugin_url: &str,
- plugin: &str,
- cid: &str,
- file_path: &str,
- existing_meta: &BTreeMap,
- ) -> anyhow::Result<()> {
- let url = format!("{}/process", plugin_url.trim_end_matches('/'));
- let body = ProcessRequest {
- task_id: format!("{plugin}-{cid}-{}", now_nanos()),
- cid,
- file_path,
- callback_url: &self.cfg.callback_url,
- meta_core_url: &self.cfg.meta_core_url,
- existing_meta,
- };
- let resp = self.http.post(&url).json(&body).send().await?;
- if !resp.status().is_success() {
- let status = resp.status();
- let text = resp.text().await.unwrap_or_default();
- anyhow::bail!("{plugin} POST {url} returned {status}: {text}");
- }
- Ok(())
- }
-
- /// POST `/configure` to the tmdb plugin with the API key + language.
- async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> {
- let Some(token) = self.cfg.tmdb_token.clone() else {
- anyhow::bail!("no TMDB token configured");
- };
- let url = format!("{}/configure", tmdb_url.trim_end_matches('/'));
- let body = serde_json::json!({
- "config": { "apiKey": token, "language": self.cfg.tmdb_language }
- });
- let resp = self.http.post(&url).json(&body).send().await?;
- if !resp.status().is_success() {
- let status = resp.status();
- let text = resp.text().await.unwrap_or_default();
- anyhow::bail!("tmdb /configure returned {status}: {text}");
- }
- Ok(())
- }
-
- /// POST `/configure` to the opensubtitles plugin with the API key, account
- /// credentials, and wanted languages.
- async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> {
- let Some(key) = self.cfg.opensubtitles_api_key.clone() else {
- anyhow::bail!("no OpenSubtitles API key configured");
- };
- let cfg_url = format!("{}/configure", url.trim_end_matches('/'));
- let body = serde_json::json!({
- "config": {
- "apiKey": key,
- "username": self.cfg.opensubtitles_username,
- "password": self.cfg.opensubtitles_password,
- "languages": self.cfg.opensubtitles_languages,
- }
- });
- let resp = self.http.post(&cfg_url).json(&body).send().await?;
- if !resp.status().is_success() {
- let status = resp.status();
- let text = resp.text().await.unwrap_or_default();
- anyhow::bail!("opensubtitles /configure returned {status}: {text}");
- }
- Ok(())
- }
-
- /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or
- /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta`
- /// and writes the `subtitles//` key-set straight to meta-core).
- /// Best-effort: any failure is logged and swallowed.
- async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) {
- let Some(url) = self.cfg.opensubtitles_url.clone() else {
- return;
- };
- if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") {
- debug!(
- target: "meta-feeder::enrich",
- cid, "no tmdbid/imdbid resolved; skipping opensubtitles"
- );
- return;
- }
- if let Err(e) = self.configure_opensubtitles(&url).await {
- warn!(
- target: "meta-feeder::enrich",
- cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)"
- );
- }
- if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await {
- warn!(
- target: "meta-feeder::enrich",
- cid, error = %e, "opensubtitles /process failed (non-fatal)"
- );
- }
- }
-
- /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout`
- /// elapses; returns the merged field map on success.
- async fn poll_until_field(
- &self,
- cid: &str,
- field: &str,
- timeout: Duration,
- ) -> Option> {
- let deadline = SystemTime::now() + timeout;
- loop {
- match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await {
- Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => {
- return Some(fields);
- }
- Ok(_) => {}
- Err(e) => debug!(
- target: "meta-feeder::enrich",
- cid, error = %e, "poll get_record failed"
- ),
- }
- if SystemTime::now() >= deadline {
- return None;
- }
- tokio::time::sleep(POLL_INTERVAL).await;
- }
- }
-}
-
-fn now_nanos() -> u128 {
- SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .map(|d| d.as_nanos())
- .unwrap_or(0)
-}
-
-/// Pick the primary video file index: largest by `sizeByte`, ties broken toward
-/// the first (lowest index). Falls back to 0 for an all-unsized bundle.
-fn pick_primary(videos: &[EnrichTarget]) -> usize {
- videos
- .iter()
- .enumerate()
- .max_by_key(|(i, v)| {
- let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0);
- // larger size wins; on a tie the smaller index wins (Reverse).
- (size, std::cmp::Reverse(*i))
- })
- .map(|(i, _)| i)
- .unwrap_or(0)
-}
-
-/// Project a resolved record down to the show-level anchor fields that are safe
-/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]).
-fn extract_anchor(fields: &BTreeMap) -> BTreeMap {
- fields
- .iter()
- .filter(|(k, _)| {
- ANCHOR_EXACT.contains(&k.as_str())
- || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p))
- })
- .map(|(k, v)| (k.clone(), v.clone()))
- .collect()
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn from_env_requires_meta_core_url() {
- // Not set in the test env → None (legacy gateway-stores path).
- // (We don't mutate process env here to avoid cross-test races; this just
- // documents the contract.)
- assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some());
- }
-
- fn tgt(cid: &str, size: u64) -> EnrichTarget {
- EnrichTarget {
- cid: cid.into(),
- file_name: cid.into(),
- base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]),
- }
- }
-
- #[test]
- fn pick_primary_is_largest_then_first() {
- let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)];
- // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins.
- assert_eq!(pick_primary(&v), 1);
- }
-
- #[test]
- fn pick_primary_all_unsized_defaults_first() {
- let v = vec![
- EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() },
- EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() },
- ];
- assert_eq!(pick_primary(&v), 0);
- }
-
- #[test]
- fn extract_anchor_keeps_show_level_drops_per_file() {
- let rec = BTreeMap::from_iter([
- ("tmdbid".to_string(), "10378".to_string()),
- ("imdbid".to_string(), "tt1254207".to_string()),
- ("poster".to_string(), "bafkPOSTER".to_string()),
- ("genres".to_string(), "Animation|Comedy".to_string()),
- ("plot/eng".to_string(), "a plot".to_string()),
- ("tags".to_string(), "tmdb-verified".to_string()),
- // per-file — must NOT be fanned:
- ("originalTitle".to_string(), "Big Buck Bunny".to_string()),
- ("season".to_string(), "1".to_string()),
- ("episode".to_string(), "3".to_string()),
- ("fileName".to_string(), "ep3.mkv".to_string()),
- ]);
- let a = extract_anchor(&rec);
- assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng"));
- assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid"));
- assert!(!a.contains_key("originalTitle"), "originalTitle is per-file");
- assert!(!a.contains_key("season") && !a.contains_key("episode"));
- assert!(!a.contains_key("fileName"));
- }
-}
diff --git a/crates/meta-feeder-sdk/src/filename_meta.rs b/crates/meta-feeder-sdk/src/filename_meta.rs
deleted file mode 100644
index 8c93c11..0000000
--- a/crates/meta-feeder-sdk/src/filename_meta.rs
+++ /dev/null
@@ -1,1309 +0,0 @@
-//! Season / episode extraction from a torrent release title.
-//!
-//! This is a Rust port of `@metazla/filename-tools`
-//! (`packages/filename-tool/src/lib/...`) — specifically the
-//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` →
-//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the
-//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the
-//! best season/episode heuristic we have (the meta-sort `torrent` plugin
-//! already uses it), so rather than invent a second, weaker heuristic for
-//! the gateway we mirror its pattern set and ordering verbatim.
-//!
-//! ## Differences from the TS original (deliberate, documented)
-//!
-//! 1. **No parent-folder fallback.** The TS extractor also reads
-//! `Season N` out of the *parent directory* name. A torznab hit is a
-//! single release title with no directory context, so that branch is
-//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup
-//! simply has nothing to read.
-//! 2. **Rightmost-number fallback uses the captured digits, not the raw
-//! regex match.** The TS `extractRightmostNumber` does
-//! `parseFloat(fullMatch)` where the match can include a leading `.`
-//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat`
-//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the
-//! bare number) instead, which is what the function *means* to return.
-//!
-//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so
-//! a movie's release year can't be misread as an episode number by the
-//! rightmost-number fallback.
-
-use std::collections::BTreeSet;
-use std::sync::LazyLock;
-
-use regex::Regex;
-
-/// Structured season/episode extracted from a release title. Each field
-/// is the stringified number exactly as the TS extractor would emit it
-/// (`"" + n`): whole numbers render without a decimal, half-episode
-/// specials keep the `.5` (e.g. `"13"`, `"13.5"`).
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
-pub struct SeasonEpisode {
- pub season: Option,
- pub episode: Option,
- /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`).
- /// Set together with [`episode_end`]; mutually exclusive with the
- /// single-episode [`episode`] field.
- pub episode_start: Option,
- /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`).
- pub episode_end: Option,
- /// `season * 10000 + episode` disambiguator (matches the TS
- /// `increment` field). Computed for completeness; the torznab caller
- /// currently only persists `season`/`episode`.
- pub increment: Option,
- /// True when this release is a **pack/batch** rather than a single
- /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season
- /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with
- /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack
- /// is deliberately **not** given a single `episode`.
- pub is_pack: bool,
- /// True when [`season`] came from an explicit season-bearing token
- /// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left
- /// season-less (no "assume season 1" default), so a per-file reparse uses
- /// this to know it should borrow the season from the pack/parent title (see
- /// [`extract_season_episode_with_context`]).
- pub season_explicit: bool,
-}
-
-impl SeasonEpisode {
- /// A pack carrying a concrete inclusive episode range.
- pub fn has_episode_range(&self) -> bool {
- self.episode_start.is_some() && self.episode_end.is_some()
- }
-}
-
-// ---------------------------------------------------------------------------
-// Pattern fragments — copied 1:1 from
-// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`.
-// ---------------------------------------------------------------------------
-
-const SW: &str = r"(?:\.|\b)"; // a dot or word boundary
-const S: &str = r"(?:\.|\b|\s)";
-const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored
-const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5)
-const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5)
-const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5)
-const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99
-const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5)
-
-/// Season-AND-episode patterns, longest/most-specific first. Each has
-/// exactly two capture groups: group 1 = season, group 2 = episode.
-/// Order matters — the first to match wins (mirrors the TS array order).
-static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| {
- [
- format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"),
- format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"),
- format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"),
- format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"),
- format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"),
- format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"),
- format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"),
- format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"),
- format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"),
- format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"),
- format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"),
- format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"),
- ]
- .iter()
- .map(|p| Regex::new(p).expect("season+episode pattern compiles"))
- .collect()
-});
-
-/// Episode-only patterns. Each has exactly one capture group: the episode.
-static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| {
- [
- format!(r"(?i){SW}Episode{S}*{EP4}{END}"),
- format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"),
- format!(r"(?i){SW}E{S}*{EP4MY}{END}"),
- format!(r"(?i){SW}Ep{EP4MY}{END}"),
- ]
- .iter()
- .map(|p| Regex::new(p).expect("episode-only pattern compiles"))
- .collect()
-});
-
-/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number.
-static SOLO_EP: LazyLock =
- LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles"));
-
-// ---------------------------------------------------------------------------
-// Episode-range / pack detection. Anime + TV batches encode a *range* of
-// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season
-// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so
-// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a
-// batch collapses to one phantom episode. We detect ranges FIRST (before
-// `find_season_and_episode`) and emit `episode_start`/`episode_end` +
-// `is_pack`, so the caller can stamp `contentKind=pack` instead.
-//
-// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`:
-// - ranges use a `-`/`~` separator with TWO numbers and `end > start`;
-// - the bare form requires the start digit to be preceded by a space/`_`/`-`
-// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read
-// as a range start — that release falls through to the single-episode path;
-// - 4-digit year-like / out-of-range operands are rejected.
-// ---------------------------------------------------------------------------
-
-/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`.
-static RANGE_BRACKET: LazyLock = LazyLock::new(|| {
- Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles")
-});
-
-/// CJK episode range: `第29-38話` / `第29~38話`.
-static RANGE_CJK: LazyLock = LazyLock::new(|| {
- Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles")
-});
-
-/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces
-/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a
-/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight
-/// separator is the disambiguator from the very common `Show 2 - 11` /
-/// `Show - 11` episode-separator convention (spaced dash → single episode, not
-/// a range); spaced ranges only come through the unambiguous bracketed form.
-static RANGE_BARE: LazyLock = LazyLock::new(|| {
- Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)")
- .expect("bare range compiles")
-});
-
-/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single
-/// season. Matched only as a fallback when no episode range is present.
-static SEASON_RANGE: LazyLock = LazyLock::new(|| {
- Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles")
-});
-
-/// `(start, end)` is a plausible inclusive episode range: ascending, and
-/// neither operand is a 4-digit year / resolution-like number.
-fn plausible_range(start: u32, end: u32) -> bool {
- end > start && (1..1900).contains(&start) && end < 1900
-}
-
-/// Detect an episode range anywhere in the (precleaned) title. Returns the
-/// first plausible `(start, end)` found, trying the unambiguous bracketed and
-/// CJK forms before the bare form.
-fn detect_episode_range(s: &str) -> Option<(u32, u32)> {
- for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] {
- for caps in re.captures_iter(s) {
- let start = caps.get(1).and_then(|m| m.as_str().parse::().ok());
- let end = caps.get(2).and_then(|m| m.as_str().parse::().ok());
- if let (Some(a), Some(b)) = (start, end) {
- if plausible_range(a, b) {
- return Some((a, b));
- }
- }
- }
- }
- None
-}
-
-/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix.
-static SFV: LazyLock =
- LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles"));
-
-// ---------------------------------------------------------------------------
-// Keyword pre-clean — copied from
-// `filename-tool/src/lib/config/KeywordsArray.ts`.
-// Removed before season/episode detection so release tags like `x264`,
-// `1080p`, `720` can't be mistaken for episode numbers by the fallback.
-// ---------------------------------------------------------------------------
-
-/// Word-bounded keywords (resolution/codec/source/group tags). Order is
-/// irrelevant once compiled individually, but we sort longest-first to
-/// match the TS constructor (`keyword.length` desc) for identical
-/// overlap behaviour.
-const KEYWORDS: &[&str] = &[
- "VF",
- "aXXo",
- "STFR",
- "SUBFORCED",
- "BrRipx264",
- "YIFY",
- "TRUEFRENCH",
- "VFF",
- "ATVP",
- "RenewalEX",
- "BD",
- "DD",
- "8bits",
- "8 bits",
- "10bits",
- "10 bits",
- "part 1",
- "part 2",
- "part 3",
- "part 4",
- "part A",
- "part B",
- "part C",
- "part D",
- "AAC5",
- "AAC5.1",
- "1080i",
- "1080p",
- "2160p",
- "480i",
- "480p",
- "4k",
- "576i",
- "576p",
- "720",
- "720i",
- "720p",
- "aac",
- "aac4",
- "ac3",
- "amzn",
- "apple tv+",
- "avi",
- "bbc",
- "bd5",
- "bdrip",
- "blu-ray",
- "bluray",
- "brrip",
- "cam",
- "cw",
- "dc",
- "dcu",
- "ddp5.1",
- "director's cut",
- "disney+",
- "divx",
- "divx5",
- "dl",
- "dsr",
- "dsrip",
- "dts",
- "dual audio",
- "dubbed",
- "dvd",
- "dvdivx",
- "dvdr",
- "dvdrip",
- "dvdscr",
- "dvdscreener",
- "eng",
- "eng sub",
- "esp",
- "fan edit",
- "fhd",
- "flv",
- "fs",
- "ger",
- "german",
- "h.264",
- "h.265",
- "h264",
- "h265",
- "hardcoded",
- "hbo",
- "hd",
- "hddvd",
- "hdr",
- "hdrip",
- "hdtv",
- "hdtvrip",
- "hevc",
- "hq",
- "hrhd",
- "hrhdtv",
- "hulu",
- "imax",
- "ita",
- "jpn",
- "ld",
- "md",
- "mkv",
- "mp3",
- "mp4",
- "mpeg",
- "mpg",
- "mq",
- "multi",
- "multisubs",
- "netflix",
- "nf",
- "nfofix",
- "ntsc",
- "ogg",
- "ogm",
- "ova",
- "pal",
- "pdtv",
- "r3",
- "r5",
- "rerip",
- "rsvcd",
- "screener",
- "sd",
- "se",
- "subbed",
- "svcd",
- "tc",
- "telecine",
- "telesync",
- "ts",
- "tv series",
- "uhd",
- "uhdtv",
- "uhdv",
- "v2",
- "vcd",
- "vostfr",
- "web",
- "web-dl",
- "webcast",
- "webrip",
- "wmv",
- "ws",
- "www",
- "x264",
- "x265",
- "xsvcd",
- "xvid",
- "xvidvd",
- "xxx",
- "bits",
- "AnimeServ",
-];
-
-/// Substrings removed without word boundaries (the TS `substringArray`).
-const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"];
-
-/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first.
-static KEYWORD_RES: LazyLock> = LazyLock::new(|| {
- let mut kws: Vec<&str> = KEYWORDS.to_vec();
- kws.sort_by_key(|k| std::cmp::Reverse(k.len()));
- kws.into_iter()
- .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles"))
- .collect()
-});
-
-/// Compiled substring regexes (no boundaries, case-insensitive), longest-first.
-static SUBSTRING_RES: LazyLock> = LazyLock::new(|| {
- let mut subs: Vec<&str> = SUBSTRINGS.to_vec();
- subs.sort_by_key(|s| std::cmp::Reverse(s.len()));
- subs.into_iter()
- .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles"))
- .collect()
-});
-
-// ---------------------------------------------------------------------------
-// Language tags — anime/torznab upstreams encode language as release-name
-// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a
-// structured field. `extract_languages` reads *every* tag off the raw title
-// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`.
-// The gateway writes each as a `languages/ = "true"` key-set member
-// (METADATA_KEYS §9) that the preferred-language filter matches against.
-//
-// Patterns are ported from the battle-tested torrent-name parsers
-// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead
-// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`,
-// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex`
-// crate has no lookaround. Two defensive choices do most of the work:
-// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They
-// collide with ordinary words, names, and domain TLDs. We rely on the
-// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead.
-// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside
-// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`.
-// ---------------------------------------------------------------------------
-
-/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD
-/// or path token can't be read as a language. Only `www.`-prefixed runs are
-/// removed — a generic `host.tld` matcher would eat dotted release names like
-/// `Spy.x.Family`. Runs on the raw (still-dotted) title.
-static URL_STRIP: LazyLock =
- LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles"));
-
-/// Guard phrases removed from the separator-normalized title before language
-/// matching, because each contains a substring that would otherwise
-/// false-positive a tag. None of these phrases is itself a language tag, so
-/// removal is loss-free for detection — this is how we reproduce the upstream
-/// parsers' lookbehind guards without lookaround:
-/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese)
-/// - `tel aviv` → guards `\btel\b` (Telugu)
-/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker)
-static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| {
- Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b")
- .expect("lang guard re compiles")
-});
-
-/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match
-/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js`
-/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`),
-/// because meta-share's "languages I read" preference is the source of truth
-/// the meta-watch filter compares against — the enriched value must be in the
-/// same space the chips use. (This deliberately diverges from `gutenberg.rs` /
-/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.)
-///
-/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for
-/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release
-/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul`
-/// and `fre`: every pattern that matches contributes its code, so the result
-/// is a set, not a first-wins pick.
-static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| {
- [
- // Multi / dual-audio → "mul". `dl` matches only after "web dl" has
- // been stripped (see LANG_GUARD_STRIP).
- (
- r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b",
- "mul",
- ),
- // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…).
- (
- r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b",
- "fre",
- ),
- (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"),
- (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"),
- (r"\bitalian\b|\bita\b", "ita"),
- (r"\bgerman\b|\bger\b|\bdeu\b", "ger"),
- (
- r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b",
- "spa",
- ),
- (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"),
- (r"\bkorean\b|\bkor\b", "kor"),
- (
- r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b",
- "chi",
- ),
- (r"\brussian\b|\brus\b", "rus"),
- (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"),
- (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"),
- (r"\bczech\b|\bcze\b|\bces\b", "cze"),
- (r"\bslovak\b|\bslo\b|\bslk\b", "slo"),
- (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"),
- (
- r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b",
- "rum",
- ),
- (r"\bgreek\b|\bgre\b|\bell\b", "gre"),
- (r"\bukrainian\b|\bukr\b", "ukr"),
- (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"),
- (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"),
- (r"\bnorwegian\b|\bnorsk\b", "nor"),
- (r"\bdanish\b|\bdansk\b", "dan"),
- (r"\bfinnish\b|\bsuomi\b", "fin"),
- (r"\bturkish\b|\btur\b", "tur"),
- (r"\barabic\b|\bara\b", "ara"),
- (r"\bhindi\b|\bhin\b", "hin"),
- (r"\btamil\b|\btam\b", "tam"),
- (r"\btelugu\b|\btel\b", "tel"),
- (r"\bmalayalam\b|\bmal\b", "mal"),
- (r"\bvietnamese\b|\bvie\b", "vie"),
- (r"\bthai\b|\btha\b", "tha"),
- ]
- .iter()
- .map(|(p, code)| {
- (
- Regex::new(&format!("(?i){p}")).expect("language tag re compiles"),
- *code,
- )
- })
- .collect()
-});
-
-/// Characters `cleanSpecialChars` replaces with a space (note: `.` and
-/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07`
-/// structure).
-const SPECIAL_CHARS: &[char] = &[
- '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"',
- '`', '~',
-];
-
-/// Extract season/episode from a release title.
-///
-/// Faithful to `computeSeasonAndEpisodeFromFilename` →
-/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath`
-/// (minus the parent-folder branch — see the module doc).
-pub fn extract_season_episode(title: &str) -> SeasonEpisode {
- let cleaned = preclean(title);
- // Range detection runs on the *raw* title (with `~` normalized to `-`),
- // not the precleaned one: `preclean` replaces `~` with a space (it's a
- // "special char"), which would erase a `26~37` range, and it never touches
- // the `(`/`[` brackets the bracketed form anchors on.
- let range_src = title.replace('~', "-");
-
- // Pack detection runs first — an episode range (`(01-10)`) would otherwise
- // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}`
- // pattern. A pack is never given a single `episode`.
- if let Some((start, end)) = detect_episode_range(&range_src) {
- let season = find_pack_season(&cleaned);
- return SeasonEpisode {
- is_pack: true,
- episode_start: Some(start.to_string()),
- episode_end: Some(end.to_string()),
- // An explicit season token (`S2 (01-10)`) buckets the pack; a bare
- // range (`Frieren (01-28)`) leaves the season for the caller.
- season: season.map(|s| s.to_string()),
- season_explicit: season.is_some(),
- ..Default::default()
- };
- }
- // Season range (`S01-S03`) — a whole-series pack, no single season/episode.
- if SEASON_RANGE.is_match(&range_src) {
- return SeasonEpisode {
- is_pack: true,
- ..Default::default()
- };
- }
-
- let (sea, ep) = find_season_and_episode(&cleaned);
-
- let mut out = SeasonEpisode::default();
- // season/episode 0 are valid; only < 0 means "not found".
- if sea >= 0.0 {
- out.season = Some(fmt_num(sea));
- out.season_explicit = true;
- }
- if ep >= 0.0 {
- out.episode = Some(fmt_num(ep));
- // An episode with no season is left season-LESS on purpose: a season is
- // only asserted when explicitly hinted — an S-token in the title here,
- // or (via extract_season_episode_with_context) the parent/pack title.
- // Absolute-numbered releases ("Show - 28") thus carry an episode and no
- // season, so the UI buckets them under "Episodes" rather than a
- // fabricated "Season 1". `increment` encodes a season, so it is only
- // emitted when the season is known.
- if sea >= 0.0 {
- out.increment = Some(fmt_num(sea * 10000.0 + ep));
- }
- } else if sea >= 0.0 {
- // Season present, no episode → a whole-season pack (`Season 3`, `S03`).
- out.is_pack = true;
- }
- out
-}
-
-/// Extract season/episode using a `/` style context — the
-/// per-file path of an opened multi-file (pack) torrent. Parses the file's own
-/// `filename` first; when the filename yields an episode but **no** season, the
-/// season is recovered from the `parent` (the pack/torrent title). This
-/// restores the parent-folder season fallback the TS
-/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped
-/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves
-/// to S2E3 rather than the inherited pack-level number.
-pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode {
- let mut file_se = extract_season_episode(filename);
- // Only borrow the parent's season for a concrete single episode whose own
- // filename carries no *explicit* season token (so it is otherwise
- // season-less). A per-file record is an episode, not a pack, so we don't
- // propagate the parent's pack/range shape here.
- if file_se.episode.is_some() && !file_se.season_explicit {
- if let Some(s) = find_pack_season(&preclean(parent)) {
- file_se.season = Some(s.to_string());
- file_se.season_explicit = true;
- if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) {
- file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep));
- }
- }
- }
- file_se
-}
-
-/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`,
-/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to
-/// recover a per-file season from the parent. Excludes the bare-number season
-/// form so an episode-only title isn't mis-bucketed.
-fn find_pack_season(s: &str) -> Option {
- let padded = format!(" {s} ");
- for re in SEASON_WORD_ONLY_PATTERNS.iter() {
- if let Some(caps) = re.captures(&padded) {
- let v = parse_group(&caps, 1);
- if v >= 0.0 {
- return Some(v as i64);
- }
- }
- }
- None
-}
-
-/// Best-effort set of languages a release supports (audio **and** subtitles,
-/// mixed), inferred from its title tags.
-///
-/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`,
-/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we
-/// read them off the raw title here. Returns the **full set** of ISO 639-2/B
-/// alpha-3 codes found (matching meta-share's UI vocabulary — see
-/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is
-/// empty when the title carries no recognizable tag — the caller turns that
-/// into the `languages/und` sentinel.
-pub fn extract_languages(title: &str) -> BTreeSet {
- // 1. Strip `www.…` domain noise while the dots are still intact.
- let stripped = URL_STRIP.replace_all(title, " ");
- // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags
- // anchor inside dotted/underscored/hyphenated release names — and so
- // the guard phrases ("web dl", "shang chi") match regardless of which
- // separator the original used.
- let norm: String = stripped
- .chars()
- .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c })
- .collect();
- // 3. Remove guard phrases that would otherwise false-positive a tag.
- let guarded = LANG_GUARD_STRIP.replace_all(&norm, " ");
- // 4. Collect every matching code (a set — dedupes, deterministic order).
- let mut out = BTreeSet::new();
- for (re, code) in LANGUAGE_TAGS.iter() {
- if re.is_match(&guarded) {
- out.insert((*code).to_string());
- }
- }
- out
-}
-
-// ---------------------------------------------------------------------------
-// Quality — the resolution *tier* of a release, normalized to a single label
-// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured
-// resolution, so — like season/episode and language — the release title is the
-// only source. Mirrors the client-side `parseVersion()` heuristic in
-// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's
-// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to
-// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height.
-// ---------------------------------------------------------------------------
-
-/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH`
-/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title
-/// carries no recognizable resolution.
-pub fn extract_quality(title: &str) -> Option {
- static TIER: LazyLock = LazyLock::new(|| {
- Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles")
- });
- static WXH: LazyLock =
- LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles"));
- // Normalize `.`/`_` separators so `\b`-anchored tokens match inside
- // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`).
- let t = title.replace(['.', '_'], " ");
- if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) {
- let tok = m.as_str().to_lowercase();
- return Some(
- match tok.as_str() {
- "4k" => "2160p",
- "1080i" => "1080p",
- "720i" => "720p",
- other => other,
- }
- .to_string(),
- );
- }
- if let Some(h) = WXH
- .captures(&t)
- .and_then(|c| c.get(1))
- .and_then(|m| m.as_str().parse::().ok())
- {
- let tier = if h >= 2000 {
- "2160p"
- } else if h >= 1000 {
- "1080p"
- } else if h >= 700 {
- "720p"
- } else if h >= 500 {
- "576p"
- } else {
- "480p"
- };
- return Some(tier.to_string());
- }
- None
-}
-
-/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12
-/// structured search facet). Title-derived / best-effort — same provenance and
-/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in
-/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens
-/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and
-/// any stream-derived value.
-pub fn extract_codec(title: &str) -> Option {
- static CODEC: LazyLock = LazyLock::new(|| {
- Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b")
- .expect("codec re compiles")
- });
- // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored
- // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`).
- let t = title.replace(['.', '_'], " ");
- let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase();
- let cleaned = tok.replace(' ', "");
- let norm = match cleaned.as_str() {
- "x265" | "h265" | "hevc" => "hevc",
- "x264" | "h264" | "avc" => "h264",
- other => other, // av1, xvid, divx, vp9
- };
- Some(norm.to_string())
-}
-
-/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
-/// `cleanSpace`, in that order.
-fn preclean(filename: &str) -> String {
- let mut s = filename.to_string();
- for re in KEYWORD_RES.iter() {
- s = re.replace_all(&s, "").into_owned();
- }
- for re in SUBSTRING_RES.iter() {
- s = re.replace_all(&s, "").into_owned();
- }
- // removeSFV — first occurrence only (TS uses a non-global replace).
- s = SFV.replace(&s, "").into_owned();
- // cleanSpecialChars — replace each special char with a space.
- s = s
- .chars()
- .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c })
- .collect();
- // cleanSpace — collapse runs of whitespace and trim.
- s.split_whitespace().collect::>().join(" ")
-}
-
-/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0`
-/// for "not found".
-fn find_season_and_episode(filename: &str) -> (f64, f64) {
- // Pad with spaces so word-boundary fragments anchor at the ends
- // exactly as the TS code does (`" " + filename + " "`).
- let padded = format!(" {filename} ");
-
- for re in SEASON_EPISODE_PATTERNS.iter() {
- if let Some(caps) = re.captures(&padded) {
- let season = parse_group(&caps, 1);
- let episode = parse_group(&caps, 2);
- return (season, episode);
- }
- }
- for re in EPISODE_PATTERNS.iter() {
- if let Some(caps) = re.captures(&padded) {
- let episode = parse_group(&caps, 1);
- return (-1.0, episode);
- }
- }
- // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught
- // here — *before* the bare rightmost-number fallback — so a whole-season
- // pack reports `(season, no-episode)` instead of the fallback mis-reading
- // its season digit as an episode and `extract_season_episode` then
- // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03`
- // bug). The TS pipeline relied on a parent-folder lookup for this case,
- // which the gateway dropped (see module doc).
- for re in SEASON_WORD_ONLY_PATTERNS.iter() {
- if let Some(caps) = re.captures(&padded) {
- let season = parse_group(&caps, 1);
- if season >= 0.0 {
- return (season, -1.0);
- }
- }
- }
- // Last hope: the rightmost bare number. `if (number)` in the TS is
- // falsy for 0, so a lone 0 doesn't count as an episode.
- if let Some(n) = rightmost_number(&padded) {
- if n != 0.0 {
- return (-1.0, n);
- }
- }
- (-1.0, -1.0)
-}
-
-fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 {
- caps.get(idx)
- .and_then(|m| m.as_str().parse::().ok())
- .unwrap_or(-1.0)
-}
-
-/// `extractRightmostNumber` — the last `soloEp` match's number.
-fn rightmost_number(s: &str) -> Option {
- SOLO_EP
- .captures_iter(s)
- .filter_map(|c| c.get(1))
- .last()
- .and_then(|m| m.as_str().parse::().ok())
-}
-
-/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64
-/// `Display` already drops the trailing `.0`, so this is a thin wrapper
-/// kept for intent.
-fn fmt_num(n: f64) -> String {
- format!("{n}")
-}
-
-// ---------------------------------------------------------------------------
-// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` +
-// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`.
-// The gateway previously used a separate hand-rolled `clean_torrent_title`
-// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and
-// season-word/episode-range stripping — so TMDB enrichment failed on many
-// release-name shapes. Reuse the already-ported building blocks instead.
-// ---------------------------------------------------------------------------
-
-/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` /
-/// `S N` / trailing bare number and everything after it.
-static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| {
- [
- format!(r"(?i){SW}Season{S}*{SEA}{END}"),
- format!(r"(?i){SW}Saison{S}*{SEA}{END}"),
- format!(r"(?i){SW}S{S}*{SEA}{END}"),
- format!(r"(?i){SW}{SEA}{END}"),
- ]
- .iter()
- .map(|p| Regex::new(p).expect("season-only pattern compiles"))
- .collect()
-});
-
-/// Season-only patterns restricted to the **keyword'd** forms (`Season N`,
-/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a
-/// whole-season *pack* (no episode) before the bare rightmost-number fallback
-/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}`
-/// entry: that one matches any lone number and would turn an episode-only
-/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because
-/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`.
-static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| {
- [
- format!(r"(?i){SW}Season{S}*{SEA}{END}"),
- format!(r"(?i){SW}Saison{S}*{SEA}{END}"),
- format!(r"(?i){SW}S{S}*{SEA}{END}"),
- ]
- .iter()
- .map(|p| Regex::new(p).expect("season-word-only pattern compiles"))
- .collect()
-});
-
-/// `cleanTags` — bracketed/parenthesised/braced groups.
-static TAG_RE: LazyLock =
- LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles"));
-/// `removeTagSeparator` — stray bracket chars.
-static TAG_SEP_RE: LazyLock =
- LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles"));
-/// `removeYear` — a standalone 19xx/20xx.
-static YEAR_RE: LazyLock =
- LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles"));
-/// `removeBeyondHyphen` — capture everything before the first ` - `
-/// (space-delimited) hyphen.
-static BEYOND_HYPHEN_RE: LazyLock =
- LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles"));
-static TRAILING_HYPHEN_RE: LazyLock =
- LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles"));
-
-fn clean_special_chars(s: &str) -> String {
- s.chars()
- .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c })
- .collect()
-}
-
-fn remove_keywords(s: &str) -> String {
- let mut out = s.to_string();
- for re in KEYWORD_RES.iter() {
- out = re.replace_all(&out, "").into_owned();
- }
- for re in SUBSTRING_RES.iter() {
- out = re.replace_all(&out, "").into_owned();
- }
- out
-}
-
-fn remove_beyond_hyphen(s: &str) -> String {
- if let Some(c) = BEYOND_HYPHEN_RE.captures(s) {
- return c
- .get(1)
- .map(|m| m.as_str().trim().to_string())
- .unwrap_or_default();
- }
- TRAILING_HYPHEN_RE.replace(s, "").trim().to_string()
-}
-
-fn remove_season_and_episode(s: &str) -> String {
- for re in SEASON_EPISODE_PATTERNS
- .iter()
- .chain(EPISODE_PATTERNS.iter())
- .chain(SEASON_ONLY_PATTERNS.iter())
- {
- if re.is_match(s) {
- return re.replace(s, "").trim().to_string();
- }
- }
- // removeRightmostNumber fallback.
- SOLO_EP.replace(s, "").trim().to_string()
-}
-
-/// Produce a clean show/movie title from a raw release name, faithful to
-/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB
-/// search — NOT a hand-rolled cleaner.
-pub fn clean_title(raw: &str) -> String {
- let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`)
- s = remove_keywords(&s); // 172 release/quality keywords + v1..v5
- s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags
- s = remove_beyond_hyphen(&s); // drop everything after the first " - "
- s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets
- s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear
- s = SFV.replace(&s, "").into_owned(); // removeSFV
- s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number
- s = s.replace('.', " "); // cleanDot
- let s = s.trim().trim_start_matches('-').trim_end_matches('-');
- s.split_whitespace().collect::>().join(" ")
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn se(title: &str) -> (Option, Option) {
- let r = extract_season_episode(title);
- (r.season, r.episode)
- }
-
- #[test]
- fn strict_sxxeyy() {
- assert_eq!(
- se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"),
- (Some("1".into()), Some("2".into()))
- );
- }
-
- #[test]
- fn dotted_release_name() {
- assert_eq!(
- se("The.Wire.S03E07.720p.BluRay.x264"),
- (Some("3".into()), Some("7".into()))
- );
- }
-
- #[test]
- fn clean_title_collapses_release_variants() {
- let ct = |s: &str| clean_title(s).to_lowercase();
- // Season-word, underscore/range, dotted+EnN, and the trailing-num
- // Nyaa shape all collapse to the bare show title (the cases that
- // were failing TMDB enrichment).
- assert_eq!(
- ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"),
- "spy x family"
- );
- assert_eq!(
- ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"),
- "spy x family"
- );
- assert_eq!(
- ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"),
- "spy x family"
- );
- assert_eq!(
- ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"),
- "spy x family"
- );
- }
-
- #[test]
- fn concise_nxnn() {
- assert_eq!(
- se("Some Show 1x05 720p x264"),
- (Some("1".into()), Some("5".into()))
- );
- }
-
- #[test]
- fn bracketed_concise() {
- assert_eq!(
- se("Some Show [2x10] HDTV"),
- (Some("2".into()), Some("10".into()))
- );
- }
-
- #[test]
- fn verbose_season_episode() {
- assert_eq!(
- se("My Show Season 2 Episode 13"),
- (Some("2".into()), Some("13".into()))
- );
- }
-
- #[test]
- fn anime_trailing_number_is_season_less() {
- // No season marker → episode via rightmost-number fallback, and NO
- // season is fabricated (absolute numbering). SFV id stripped, brackets
- // ignored. The UI buckets a season-less episode under "Episodes".
- assert_eq!(
- se("[SubsPlease] Some Anime - 117 [FC412C51]"),
- (None, Some("117".into()))
- );
- }
-
- #[test]
- fn half_episode_special_is_preserved() {
- assert_eq!(
- se("Some Anime S01E13.5"),
- (Some("1".into()), Some("13.5".into()))
- );
- }
-
- #[test]
- fn increment_is_season_times_10000_plus_episode() {
- let r = extract_season_episode("Show S02E05");
- assert_eq!(r.increment.as_deref(), Some("20005"));
- }
-
- #[test]
- fn four_digit_year_is_not_misread_as_episode() {
- // The `soloEp` fallback caps at 3 digits, so a 4-digit release
- // year can't be picked up — no episode here.
- let r = extract_season_episode("The Matrix 1999 1080p BluRay x264");
- assert_eq!(r.episode, None);
- }
-
- #[test]
- fn short_number_in_movie_title_is_grabbed_by_fallback() {
- // A bare 1-3 digit number in a movie title DOES trip the
- // rightmost-number fallback (here: "13"). This is exactly why the
- // torznab caller gates season/episode on TV-ness — a movie record
- // must not inherit this bogus episode.
- let r = extract_season_episode("Apollo 13 1080p BluRay x264");
- assert_eq!(r.episode.as_deref(), Some("13"));
- }
-
- #[test]
- fn keyword_strip_prevents_codec_false_positive() {
- // "x264" must be removed so "264" isn't read as the episode.
- let r = extract_season_episode("Plain Movie Title x264");
- assert_eq!(r.episode, None);
- }
-
- #[test]
- fn no_numbers_yields_nothing() {
- let r = extract_season_episode("Just A Plain Title");
- assert_eq!(r.season, None);
- assert_eq!(r.episode, None);
- }
-
- #[test]
- fn season_only_pack_is_not_misread_as_episode() {
- // Whole-season packs (no episode) must report the season with NO
- // episode — not `S01E0N` from the rightmost-number fallback. This is
- // the original bug: `Spy x Family Season 3 (S03)` was parsed as
- // season 1, episode 3, dumping the S3 pack into the Season-1 bucket.
- assert_eq!(
- se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"),
- (Some("3".into()), None)
- );
- assert_eq!(
- se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"),
- (Some("2".into()), None)
- );
- // Bare `S03` pack (no `Season` word) via the S-prefixed pattern.
- assert_eq!(
- se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"),
- (Some("3".into()), None)
- );
- assert_eq!(
- se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"),
- (Some("3".into()), None)
- );
- // French season word.
- assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None));
- }
-
- #[test]
- fn explicit_sxxeyy_still_wins_over_season_only() {
- // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected
- // by the new season-only branch.
- assert_eq!(
- se("Spy x Family Season 3 S03E13 1080p"),
- (Some("3".into()), Some("13".into()))
- );
- }
-
- #[test]
- fn episode_range_is_a_pack_not_sxxeyy() {
- // The headline bug: `(01-10)` was mis-read as S1 E10 by the
- // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no
- // single `episode`.
- let r = extract_season_episode("Frieren (01-28) [Batch] 1080p");
- assert!(r.is_pack);
- assert_eq!(r.episode, None);
- assert_eq!(r.episode_start.as_deref(), Some("1"));
- assert_eq!(r.episode_end.as_deref(), Some("28"));
-
- // Bare, dash-delimited, leading-zero range.
- let r = extract_season_episode("Some Anime 01-12 1080p");
- assert!(r.is_pack);
- assert_eq!(r.episode_start.as_deref(), Some("1"));
- assert_eq!(r.episode_end.as_deref(), Some("12"));
-
- // Tilde range, no leading zero.
- let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]");
- assert!(r.is_pack);
- assert_eq!(r.episode_start.as_deref(), Some("26"));
- assert_eq!(r.episode_end.as_deref(), Some("37"));
- }
-
- #[test]
- fn explicit_season_buckets_a_pack() {
- // `S2 (01-10)` → pack in season 2, episodes 1–10.
- let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)");
- assert!(r.is_pack);
- assert_eq!(r.season.as_deref(), Some("2"));
- assert_eq!(r.episode_start.as_deref(), Some("1"));
- assert_eq!(r.episode_end.as_deref(), Some("10"));
- }
-
- #[test]
- fn cjk_episode_range_is_a_pack() {
- let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]");
- assert!(r.is_pack);
- assert_eq!(r.episode_start.as_deref(), Some("29"));
- assert_eq!(r.episode_end.as_deref(), Some("38"));
- }
-
- #[test]
- fn season_range_is_a_pack_with_no_single_season() {
- let r = extract_season_episode("Frieren S01-S03 Complete 1080p");
- assert!(r.is_pack);
- assert_eq!(r.season, None);
- assert_eq!(r.episode, None);
- assert_eq!(r.episode_start, None);
- }
-
- #[test]
- fn whole_season_release_is_a_pack() {
- // A season-only release (`Season 3`, `S03`, no episode) is a pack too —
- // the original "season-level release shows as one media card" bug.
- let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)");
- assert!(r.is_pack);
- assert_eq!(r.season.as_deref(), Some("3"));
- assert_eq!(r.episode, None);
- }
-
- #[test]
- fn single_episode_is_not_a_pack() {
- // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all
- // single episodes, never packs.
- for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] {
- assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack");
- }
- let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]");
- assert!(!r.is_pack);
- assert_eq!(r.season.as_deref(), Some("2"));
- assert_eq!(r.episode.as_deref(), Some("7"));
-
- // The `Show - ` anime convention (spaced dash) is a
- // single episode, NOT a `2..11` range — the tight-separator rule keeps
- // this out of pack detection.
- let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]");
- assert!(!r.is_pack);
- assert_eq!(r.season.as_deref(), Some("2"));
- assert_eq!(r.episode.as_deref(), Some("11"));
-
- // A year range in a movie title is not an episode range.
- assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack);
- }
-
- #[test]
- fn per_file_context_recovers_season_from_parent() {
- // Opened pack: the inner file carries the episode, the pack title the
- // season. `Frieren - 03.mkv` alone is season-less; with the S2 pack
- // parent it resolves to S2E3 (the parent is the "season hint").
- let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv");
- assert_eq!(r.season.as_deref(), Some("2"));
- assert_eq!(r.episode.as_deref(), Some("3"));
- assert!(!r.is_pack);
-
- // The file's own explicit season wins over the parent.
- let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv");
- assert_eq!(r.season.as_deref(), Some("1"));
- assert_eq!(r.episode.as_deref(), Some("5"));
- }
-
- #[test]
- fn absolute_numbered_anime_is_season_less() {
- // No season keyword → the rightmost-number fallback still finds the
- // episode, but NO season is fabricated. Season is only ever asserted
- // from an explicit token (title or parent), never inferred — so an
- // absolute-numbered release lands in the UI's "Episodes" bucket.
- assert_eq!(
- se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"),
- (None, Some("37".into()))
- );
- }
-
- fn langs(s: &str) -> BTreeSet {
- extract_languages(s)
- }
-
- #[test]
- fn language_from_release_tags() {
- // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi).
- assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre"));
- assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger"));
- assert!(langs("Show ENG SUB 720p").contains("eng"));
- assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn"));
- assert!(langs("Film TRUEFRENCH 1080p").contains("fre"));
- }
-
- #[test]
- fn multi_is_mul_and_collects_every_language() {
- // `languages` is the union of all supported audio+sub languages.
- // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is
- // ALSO collected (the set, not a first-wins pick).
- assert_eq!(
- langs("Show MULTI VOSTFR 1080p"),
- BTreeSet::from(["mul".to_string(), "fre".to_string()])
- );
- assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul"));
- assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul"));
- // A bilingual release collects both specific codes.
- assert_eq!(
- langs("Movie.2021.GERMAN.ENGLISH.1080p"),
- BTreeSet::from(["ger".to_string(), "eng".to_string()])
- );
- }
-
- #[test]
- fn no_language_tag_yields_empty_set() {
- // A plain quality/codec release name carries no language tag — the
- // caller maps the empty set to the `languages/und` sentinel.
- assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty());
- // Short codes embedded in words/names must not false-positive: "ita"
- // inside "digital", "por" inside "portal", etc.
- assert!(langs("VFX Reel 2024 1080p").is_empty());
- assert!(langs("Digital Capital Portal 1080p").is_empty());
- }
-
- #[test]
- fn lookaround_guards_reproduced_in_code() {
- // WEB-DL must not read as `dl`→mul (German dual-language marker).
- assert!(!langs("Movie 2021 WEB-DL x264").contains("mul"));
- // "Shang-Chi" must not read as `chi` (Chinese)…
- assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi"));
- // …but a real CHI tag still does.
- assert!(langs("Movie.2021.CHI.1080p").contains("chi"));
- // "Tel Aviv" must not read as `tel` (Telugu).
- assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel"));
- // www. domain noise (incl. its TLD) must not contribute a language.
- assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty());
- }
-
- #[test]
- fn quality_tiers_and_normalization() {
- let q = |s: &str| extract_quality(s);
- assert_eq!(
- q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(),
- Some("1080p")
- );
- assert_eq!(
- q("The.Wire.S03E07.720p.BluRay.x264").as_deref(),
- Some("720p")
- );
- // 4k / interlaced collapse to the canonical progressive tier.
- assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p"));
- assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p"));
- assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p"));
- // WxH fallback, mapped by height; dot/underscore separators handled.
- assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p"));
- assert_eq!(q("Show 1280x720").as_deref(), Some("720p"));
- // No resolution token → None (never guess).
- assert_eq!(q("Just A Plain Title"), None);
- }
-
- #[test]
- fn codec_normalization() {
- let c = |s: &str| extract_codec(s);
- // HEVC encoder spellings collapse to the codec name.
- assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc"));
- assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc"));
- assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc"));
- // AVC encoder spellings collapse to h264.
- assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264"));
- assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264"));
- // Others kept as-is.
- assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1"));
- assert_eq!(c("Old Release XviD").as_deref(), Some("xvid"));
- // No codec token → None (never guess).
- assert_eq!(c("Just A Plain Title 1080p"), None);
- }
-}
diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs
deleted file mode 100644
index 2ed0cde..0000000
--- a/crates/meta-feeder-sdk/src/hash.rs
+++ /dev/null
@@ -1,1076 +0,0 @@
-//! Content-addressed identifiers used by feeder plugins' `compute_outcomes`
-//! to hash upstream-fetched bytes.
-//!
-//! Two function families live here:
-//!
-//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] —
-//! the size-prefix-plus-middle-1MB-sample identification primitive shared
-//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`.
-//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the
-//! middle slice directly so plugins that can only fetch a partial range
-//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file
-//! just to throw most of it away. **Rust-only by construction.**
-//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full
-//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec
-//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches
-//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144
-//! --hash=sha2-256 `.
-//!
-//! Cross-implementation parity for `compute_midhash256` with the TypeScript
-//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression
-//! silently breaks dedupe across the platform.
-
-use bytes::Bytes;
-use sha2::{Digest, Sha256};
-
-/// Sample window in bytes (1 MiB). Files this size or smaller are hashed
-/// in full; larger files are hashed over the middle 1 MiB slice.
-const SAMPLE_SIZE: usize = 1024 * 1024;
-
-/// Per-block IPFS chunk size (256 KiB), matches kubo's
-/// `--chunker=size-262144` default.
-pub const IPFS_CHUNK_SIZE: usize = 256 * 1024;
-
-/// dag-pb fanout — children per internal node. Matches kubo's balanced
-/// builder default.
-pub const IPFS_FANOUT: usize = 174;
-
-/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every
-/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by
-/// their own CID. Pass `.blocks` to the gateway's blockstore so peers
-/// can fetch the file by CID via bitswap.
-#[derive(Debug, Clone)]
-pub struct IpfsBlocks {
- /// Canonical "bafy…" / "bafk…" root cid string. Identical to what
- /// [`compute_ipfs_cid`] returns for the same input.
- pub root: String,
- /// Every block — root included — keyed by its own cid. For a
- /// single-leaf file, this is a one-entry vec `[(root, payload)]`
- /// (raw codec, payload is the file bytes). For multi-leaf files,
- /// it is `leaves ++ internal_nodes` in build order; the **last**
- /// entry is the root.
- pub blocks: Vec<(String, Bytes)>,
-}
-
-/// Compute a midhash256 CID matching the TypeScript implementation in
-/// `meta-hash/src/lib/file-id/FastHash.ts`:
-///
-/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the
-/// whole file if it fits in 1 MiB).
-/// 2. SHA-256 it.
-/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field
-/// and the multihash hash-code field, length 32, then multibase-base32-lower
-/// with the `b` prefix.
-///
-/// **Endianness for the size prefix is big-endian** to match TS's
-/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every
-/// import — keep the test pinned.
-pub fn compute_midhash256(bytes: &[u8]) -> String {
- let size = bytes.len();
- let sample: &[u8] = if size <= SAMPLE_SIZE {
- bytes
- } else {
- let start = (size - SAMPLE_SIZE) / 2;
- &bytes[start..start + SAMPLE_SIZE]
- };
- compute_midhash256_from_sample(size as u64, sample)
-}
-
-/// Same midhash256 primitive as [`compute_midhash256`], but takes the
-/// sample bytes directly rather than slicing them out of a full file.
-/// For upstreams that can only deliver a middle range (BitTorrent BEP 9,
-/// sliced HTTP), this avoids forcing the caller to buffer the whole file
-/// in memory just to throw most of it away.
-///
-/// `total_size` is the **full file size in bytes** — used as the
-/// size prefix, NOT `middle_sample.len()`.
-pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String {
- let size_be = total_size.to_be_bytes();
- let mut hasher = Sha256::new();
- hasher.update(size_be);
- hasher.update(middle_sample);
- let digest: [u8; 32] = hasher.finalize().into();
-
- // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20].
- // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest]
- let mut wire = Vec::with_capacity(38);
- wire.push(0x01);
- wire.extend_from_slice(&[0x80, 0x20]);
- wire.extend_from_slice(&[0x80, 0x20]);
- wire.push(0x20);
- wire.extend_from_slice(&digest);
-
- format!("b{}", base32_lower_no_padding(&wire))
-}
-
-/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1.
-///
-/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so
-/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11),
-/// len=20, digest=`.
-pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String {
- // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest]
- let mut wire = Vec::with_capacity(4 + 20);
- wire.push(0x01); // CIDv1 version
- wire.push(0x55); // codec: raw
- wire.push(0x11); // multihash: sha1
- wire.push(0x14); // multihash length: 20
- wire.extend_from_slice(infohash_20);
- format!("b{}", base32_lower_no_padding(&wire))
-}
-
-/// Custom multicodec for a single file inside a **BitTorrent v1** torrent
-/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`.
-pub const BTIH_V1_FILE_CODEC: u64 = 0x1001;
-
-/// Custom multicodec for a single file inside a **BitTorrent v2** torrent
-/// (`btih-v2-file`).
-pub const BTIH_V2_FILE_CODEC: u64 = 0x1002;
-
-/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent,
-/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info`
-/// dict) plus the zero-based file index in `info.files`. Single-file
-/// torrents use index `0`.
-pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String {
- bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index)
-}
-
-/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent
-/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's
-/// canonical traversal index in `file tree`.
-pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String {
- bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index)
-}
-
-/// Shared encoder for the two torrent-file CID families. `infohash` is 20
-/// bytes for v1 / 32 for v2; `codec` selects the family.
-fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String {
- // digest = infohash ‖ varint(file_index)
- let mut digest = Vec::with_capacity(infohash.len() + 2);
- digest.extend_from_slice(infohash);
- write_pb_varint(file_index, &mut digest);
-
- // CIDv1: [version][codec varint][multihash code varint][len varint][digest]
- let mut wire = Vec::with_capacity(1 + 4 + digest.len());
- wire.push(0x01);
- write_pb_varint(codec, &mut wire);
- write_pb_varint(codec, &mut wire);
- write_pb_varint(digest.len() as u64, &mut wire);
- wire.extend_from_slice(&digest);
- format!("b{}", base32_lower_no_padding(&wire))
-}
-
-/// Custom multicodec for a self-describing **Newznab release** locator
-/// (`nzb-release`). MetaMesh-private, adjacent to the torrent-file codecs in the
-/// `0x10xx` range. Unlike a content hash, the cid *embeds* the indexer host +
-/// release id in an identity multihash, so the credentialed meta-share peer
-/// decodes `{host,id}` straight from the cid and grabs the `.nzb` via `t=get`
-/// only at playback — no side-table, no `.nzb` download at search time. (The
-/// `0x1004` `sha256(host‖id)` opaque hash and the `0x1003` `nzb-posting`
-/// variant were both removed; only this self-describing form is emitted.)
-pub const NZB_RELEASE_CODEC: u64 = 0x1005;
-
-/// Digest byte ceiling for an `nzb-release` cid, matching meta-share's
-/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash.
-const NZB_RELEASE_MAX_DIGEST: usize = 64;
-
-/// Encode a self-describing `nzb-release` CID from a Newznab indexer's host +
-/// bare release id (the hex `` id). The multihash is *identity* (`0x00`)
-/// and its digest embeds the locator as `varint(host_len) ‖ host ‖ id_bytes`,
-/// where `id_bytes` is the hex-decoded id. meta-share's `decode_nzb_release_cid`
-/// is the exact inverse. Host-namespaced: the same release on two indexers
-/// yields different cids.
-///
-/// Returns `None` when the id isn't hex or the digest would overflow the
-/// multihash budget (an oversized host) — the caller drops that row.
-pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option {
- let id_bytes = decode_hex_id(release_id)?;
- let host_b = api_base.as_bytes();
-
- // identity-multihash digest = varint(host_len) ‖ host ‖ id_bytes
- let mut digest = Vec::with_capacity(2 + host_b.len() + id_bytes.len());
- write_pb_varint(host_b.len() as u64, &mut digest);
- digest.extend_from_slice(host_b);
- digest.extend_from_slice(&id_bytes);
- if digest.len() > NZB_RELEASE_MAX_DIGEST {
- return None;
- }
-
- // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest]
- let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
- wire.push(0x01);
- write_pb_varint(NZB_RELEASE_CODEC, &mut wire);
- wire.push(0x00); // multihash code: identity
- write_pb_varint(digest.len() as u64, &mut wire);
- wire.extend_from_slice(&digest);
- Some(format!("b{}", base32_lower_no_padding(&wire)))
-}
-
-/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the
-/// id is all-hex by construction; odd-length ids are left-padded with a `0`
-/// nibble. Returns `None` on any non-hex byte (defensive).
-fn decode_hex_id(id: &str) -> Option> {
- let padded;
- let s: &str = if id.len() % 2 == 1 {
- padded = format!("0{id}");
- &padded
- } else {
- id
- };
- let bytes = s.as_bytes();
- let mut out = Vec::with_capacity(bytes.len() / 2);
- for pair in bytes.chunks(2) {
- let hi = (pair[0] as char).to_digit(16)?;
- let lo = (pair[1] as char).to_digit(16)?;
- out.push(((hi << 4) | lo) as u8);
- }
- Some(out)
-}
-
-/// Custom multicodec for a **card locator** — the identity of a *work* (a
-/// series, a film) as published by a metadata bridge, rather than of any
-/// bytes. MetaMesh-private, the next free slot after `0x1006` url. (`0x1004`
-/// nzb-release-hash stays **retired — do not reuse**. `0x1003` was retired too,
-/// but has since been **revived** for the self-scanned Usenet posting identity —
-/// see [`NZB_POSTING_CODEC`] and `METADATA_KEYS.md` §2.)
-///
-/// This is the first CID family in the platform where **no bytes exist
-/// anywhere, ever** — `0x1005`/`0x1006` are locators too, but both eventually
-/// resolve to bytes (NNTP, HTTP), and `0x1000` midhash256 is a digest over
-/// real file bytes. A card is pure identity: the record *is* the payload, and
-/// it lives in meta-core. Consequences, all already handled:
-///
-/// * Bitswap seeding is gated on `HashKind::Sha2_256` (gateway invariant 6),
-/// so a card is never offered to the swarm — nothing to seed.
-/// * meta-share's byte path 404s for it, which is correct.
-/// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for
-/// `canonical_cid` — which is what makes the future "one card, two CIDs"
-/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe.
-pub const CARD_LOCATOR_CODEC: u64 = 0x1007;
-
-/// Digest byte ceiling for a card locator, matching meta-share's
-/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash.
-const CARD_LOCATOR_MAX_DIGEST: usize = 64;
-
-/// Encode a self-describing **card locator** CID from a metadata source and
-/// that source's own id for the work (`("tmdb", "tv:95479")`,
-/// `("mal", "52991")`). The multihash is *identity* (`0x00`) and its digest
-/// embeds the locator as `varint(source_len) ‖ source ‖ id`, both UTF-8 —
-/// exactly the shape [`compute_nzb_release_cid`] uses, minus the hex decode
-/// (a card id is arbitrary text, not a hex release id).
-///
-/// **The CID is a pure function of `(source, id)`**, which is the whole point:
-/// any peer that knows a tmdb id can derive the card's address offline and
-/// look it up locally, with no discovery round-trip. It also means the same
-/// card discovered by two different peers converges on one meta-core record
-/// for free, via the `cids/` reverse index.
-///
-/// Source-namespaced: the same work on TMDB and on MyAnimeList yields
-/// different cids. Merging those two into one record is a meta-core alias
-/// operation driven by an explicit cross-source id mapping — never inferred
-/// here (see `meta-gateway/docs/others/card-tier-search.md` §9).
-///
-/// Returns `None` when `source` is empty or the digest would overflow the
-/// multihash budget — the caller drops that card.
-pub fn compute_card_cid(source: &str, id: &str) -> Option {
- if source.is_empty() || id.is_empty() {
- return None;
- }
- let source_b = source.as_bytes();
- let id_b = id.as_bytes();
-
- // identity-multihash digest = varint(source_len) ‖ source ‖ id
- let mut digest = Vec::with_capacity(2 + source_b.len() + id_b.len());
- write_pb_varint(source_b.len() as u64, &mut digest);
- digest.extend_from_slice(source_b);
- digest.extend_from_slice(id_b);
- if digest.len() > CARD_LOCATOR_MAX_DIGEST {
- return None;
- }
-
- // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest]
- let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
- wire.push(0x01);
- write_pb_varint(CARD_LOCATOR_CODEC, &mut wire);
- wire.push(0x00); // multihash code: identity
- write_pb_varint(digest.len() as u64, &mut wire);
- wire.extend_from_slice(&digest);
- Some(format!("b{}", base32_lower_no_padding(&wire)))
-}
-
-// ---------------------------------------------------------------------------
-// Delegated-playback locators — `yt-video` (0x1008) and `ext-play` (0x1009)
-// ---------------------------------------------------------------------------
-//
-// These two are the limit case *beyond* `card` (0x1007). `card` addresses a
-// work with no bytes anywhere; these address a **rendition whose bytes exist
-// but are permanently someone else's**. An external player renders the media,
-// nothing is transported, nothing is content-addressed, nothing is re-seedable.
-//
-// See `docs/cid-formats.md` §7 and
-// `docs/study/listenbrainz-youtube-playback-tier-2026-08-21.md`.
-
-/// Custom multicodec for a **YouTube rendition** — delegated playback.
-///
-/// Its own codec rather than a `url` (`0x1006`) for three reasons, in
-/// descending order of how much trouble the alternative causes:
-///
-/// * **`0x1006` means "fetch these bytes once and seed them".** Point a `url`
-/// locator at a `watch` page and meta-share does exactly that: fetches the
-/// HTML, chunks it, seeds it, and links the resulting content CID back onto
-/// the record *as the file*. That is the MSR1 poisoning shape, except
-/// permanent and replicated across peers. A resolved `googlevideo` URL is no
-/// better — time-limited and IP-bound, so it is not a locator at all.
-/// * **The id is canonical, a URL is not.** `youtube.com` / `music.youtube.com`
-/// / `youtu.be` all address the same video, and query parameters are tracking
-/// noise, so the same rendition would mint several different CIDs. Being a
-/// pure function of `(kind, id)`, this codec makes two peers that resolve the
-/// same track converge on one meta-core record for free, via the
-/// `cids/` reverse index — the property that makes [`compute_card_cid`]
-/// useful, for the same reason.
-/// * **It is tiny** — 11 bytes of id plus a short kind prefix.
-///
-/// ⚠ **An immutable id is not durable playback.** Takedowns, geo-blocks and
-/// embed-disabled uploads all kill a reference whose CID stays valid forever —
-/// the same rot class as [`NZB_RELEASE_CODEC`]. The mitigation is at the
-/// *record* level (carry several ranked references per recording so a client
-/// can fall through), never at the CID level.
-pub const YT_VIDEO_CODEC: u64 = 0x1008;
-
-/// Custom multicodec for an **external page to open, never fetch**.
-///
-/// Wire-identical to `url` (`0x1006`) apart from the codec, and that difference
-/// is the entire contract: `0x1006` resolves cache-through and lands in the
-/// blockstore, `0x1009` **must be refused by the byte path** and only ever
-/// surfaces as an "Open on …↗" affordance. Use it for anything worth linking
-/// but not worth a per-provider codec (Bandcamp, Jamendo, an arbitrary stream
-/// page).
-pub const EXT_PLAY_CODEC: u64 = 0x1009;
-
-/// Encode a [`YT_VIDEO_CODEC`] locator from a `kind` and a bare YouTube id.
-///
-/// `kind` is `"video"` | `"playlist"` | `"channel"`; `id` is the bare
-/// identifier (`4D7u5KF7SP8`, `OLAK5uy_…`, `UCRr1xG_2WIDs18a6cIiCxeA`) — **not**
-/// a URL, and not a `watch?v=` fragment. The framing is byte-identical to
-/// [`compute_card_cid`] on purpose, so each of the seven rank/decode
-/// implementations extends a parser it already wrote rather than learning a
-/// novel shape.
-///
-/// Returns `None` on an empty field or a digest past the multihash budget —
-/// the caller drops that reference.
-pub fn compute_yt_video_cid(kind: &str, id: &str) -> Option {
- kinded_locator_cid(YT_VIDEO_CODEC, kind, id)
-}
-
-/// Encode an [`EXT_PLAY_CODEC`] locator from an `http(s)` URL.
-///
-/// The scheme check is not cosmetic: it is what keeps a `javascript:` or
-/// `data:` payload from ever reaching a client that will hand this string to
-/// an anchor or a new tab.
-pub fn compute_ext_play_cid(url: &str) -> Option {
- if !(url.starts_with("http://") || url.starts_with("https://")) {
- return None;
- }
- let digest = url.as_bytes();
- if digest.is_empty() || digest.len() > CARD_LOCATOR_MAX_DIGEST {
- return None;
- }
- Some(identity_locator_cid(EXT_PLAY_CODEC, digest))
-}
-
-/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used
-/// by [`compute_card_cid`] and [`compute_yt_video_cid`].
-fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option {
- if kind.is_empty() || id.is_empty() {
- return None;
- }
- let kind_b = kind.as_bytes();
- let id_b = id.as_bytes();
-
- let mut digest = Vec::with_capacity(2 + kind_b.len() + id_b.len());
- write_pb_varint(kind_b.len() as u64, &mut digest);
- digest.extend_from_slice(kind_b);
- digest.extend_from_slice(id_b);
- if digest.len() > CARD_LOCATOR_MAX_DIGEST {
- return None;
- }
- Some(identity_locator_cid(codec, &digest))
-}
-
-/// CIDv1 with an *identity* multihash:
-/// `[version=0x01][codec varint][mh=0x00][len varint][digest]`, base32lower.
-fn identity_locator_cid(codec: u64, digest: &[u8]) -> String {
- let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
- wire.push(0x01);
- write_pb_varint(codec, &mut wire);
- wire.push(0x00); // multihash code: identity
- write_pb_varint(digest.len() as u64, &mut wire);
- wire.extend_from_slice(digest);
- format!("b{}", base32_lower_no_padding(&wire))
-}
-
-/// Custom multicodec for a **Usenet posting** identity (`nzb-posting`),
-/// minted from the article Message-IDs of a release we scanned ourselves.
-///
-/// Unlike [`NZB_RELEASE_CODEC`] (`0x1005`), this is **not** a locator: it
-/// embeds no indexer host, so any peer with a plain NNTP provider can fetch the
-/// articles — no `IndexerCred`, no `t=get` grab. That portability is the whole
-/// point of running our own header scanner.
-///
-/// It is also **not** fetchable by cid: the digest is over the Message-ID set,
-/// not over any block's bytes, so bitswap can never satisfy a want for it (the
-/// receiver derives a block's cid by hashing the block — see
-/// `beetswap::incoming_stream`). The `.nzb` manifest travels separately, as an
-/// ordinary sha2-256 IPFS cid in the record's `manifest` field. See
-/// `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5.
-pub const NZB_POSTING_CODEC: u64 = 0x1003;
-
-/// Mint an `nzb-posting` cid (`0x1003`) from a release's article Message-IDs.
-///
-/// # The normalisation rule IS the dedup contract
-///
-/// Mirrored verbatim in meta-share (`nzb/manifest.rs`), which re-derives this
-/// digest from the fetched manifest and rejects a mismatch. **Any change here
-/// must land on both sides together** — a drift silently makes every posting
-/// unplayable. The rule:
-///
-/// 1. Trim each id, and strip a surrounding `<…>` pair if present (NZB
-/// `` bodies carry the bare form, but be defensive — NNTP `BODY`
-/// re-adds the brackets).
-/// 2. Drop empties.
-/// 3. Deduplicate, then **sort** lexicographically by bytes.
-/// 4. Join with `\n` and SHA-256 the result.
-///
-/// Case is **preserved**: a Message-ID's local part is case-sensitive per RFC
-/// 5322, so lowercasing could collide two genuinely distinct articles.
-///
-/// Sorting is what makes the cid survive segment/file reordering between two
-/// NZBs of the same posting — the reason we hash the *id set* and never the raw
-/// `.nzb` bytes (the retired `0x1004` did the latter and broke dedup; see
-/// meta-share `nzb/mod.rs`).
-///
-/// **v1 scope:** every segment counts, par2 volumes included. Two NZBs for one
-/// posting that disagree about whether to bundle par2 therefore mint different
-/// cids. Acceptable while we are the only producer of these manifests; if a
-/// second producer ever appears, the fix is to filter par2 subjects here *and*
-/// in meta-share's mirror — do not do it on one side only.
-pub fn compute_nzb_posting_cid>(message_ids: &[S]) -> String {
- let mut ids: Vec<&str> = message_ids
- .iter()
- .map(|s| {
- let t = s.as_ref().trim();
- t.strip_prefix('<')
- .and_then(|r| r.strip_suffix('>'))
- .unwrap_or(t)
- })
- .filter(|s| !s.is_empty())
- .collect();
- ids.sort_unstable();
- ids.dedup();
-
- let mut hasher = Sha256::new();
- for (i, id) in ids.iter().enumerate() {
- if i > 0 {
- hasher.update(b"\n");
- }
- hasher.update(id.as_bytes());
- }
- let digest: [u8; 32] = hasher.finalize().into();
-
- // CIDv1: [version=0x01][codec varint][mh code=0x12 sha2-256][len=32][digest]
- let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
- wire.push(0x01);
- write_pb_varint(NZB_POSTING_CODEC, &mut wire);
- wire.push(0x12); // multihash code: sha2-256
- write_pb_varint(digest.len() as u64, &mut wire);
- wire.extend_from_slice(&digest);
- format!("b{}", base32_lower_no_padding(&wire))
-}
-
-/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's
-/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144
-/// --hash=sha2-256 `.
-pub fn compute_ipfs_cid(bytes: &[u8]) -> String {
- compute_ipfs_blocks(bytes).root
-}
-
-/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every
-/// block produced along the way (leaf chunks + internal dag-pb nodes)
-/// so callers can populate a bitswap blockstore in one pass.
-pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks {
- // Leaves: chunk bytes into raw-codec blocks. Empty input still gets
- // one (empty) leaf so the cid is well-defined.
- let mut blocks: Vec<(String, Bytes)> = Vec::new();
- let leaves: Vec<(Vec, u64)> = if bytes.is_empty() {
- let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b""));
- blocks.push((cid_string(&cid_wire), Bytes::new()));
- vec![(cid_wire, 0)]
- } else {
- bytes
- .chunks(IPFS_CHUNK_SIZE)
- .map(|chunk| {
- let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk));
- blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk)));
- (cid_wire, chunk.len() as u64)
- })
- .collect()
- };
-
- // Single-leaf file: return the leaf cid as the file cid.
- if leaves.len() == 1 {
- let root = cid_string(&leaves[0].0);
- return IpfsBlocks { root, blocks };
- }
-
- // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes.
- let mut level: Vec<(Vec, u64, u64)> = leaves
- .into_iter()
- .map(|(cid, size)| (cid, size, size))
- .collect();
-
- while level.len() > 1 {
- let mut next: Vec<(Vec, u64, u64)> =
- Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT));
- for batch in level.chunks(IPFS_FANOUT) {
- let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect();
- let filesize: u64 = blocksizes.iter().sum();
- let unixfs_data = encode_unixfs_file(filesize, &blocksizes);
- let node_bytes = encode_dagpb_node(batch, &unixfs_data);
- let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes));
- blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone())));
- let own_tsize: u64 =
- batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64;
- next.push((node_cid, filesize, own_tsize));
- }
- level = next;
- }
- let root = cid_string(&level[0].0);
- IpfsBlocks { root, blocks }
-}
-
-fn cid_string(wire: &[u8]) -> String {
- format!("b{}", base32_lower_no_padding(wire))
-}
-
-/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code
-/// varint=0x12 sha2-256][len=0x20][digest...]`.
-fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec {
- debug_assert!(
- codec < 0x80,
- "codec varint must fit in one byte for this helper"
- );
- let mut wire = Vec::with_capacity(36);
- wire.push(0x01); // CIDv1
- wire.push(codec);
- wire.push(0x12); // multihash code: sha2-256
- wire.push(0x20); // digest length: 32
- wire.extend_from_slice(digest);
- wire
-}
-
-fn sha2_256(bytes: &[u8]) -> [u8; 32] {
- Sha256::digest(bytes).into()
-}
-
-/// LEB128 unsigned varint encoding into `out`.
-fn write_pb_varint(value: u64, out: &mut Vec) {
- let mut v = value;
- while v >= 0x80 {
- out.push((v as u8) | 0x80);
- v >>= 7;
- }
- out.push(v as u8);
-}
-
-fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) {
- write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint)
- write_pb_varint(value, out);
-}
-
-fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) {
- write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited
- write_pb_varint(value.len() as u64, out);
- out.extend_from_slice(value);
-}
-
-/// UnixFS protobuf payload for a File node: `Type=File, filesize,
-/// blocksizes[]`.
-fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec {
- let mut out = Vec::new();
- write_pb_varint_field(1, 2, &mut out); // Type = File
- write_pb_varint_field(3, filesize, &mut out); // filesize
- for &bs in blocksizes {
- write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated)
- }
- out
-}
-
-/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order
-/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second.
-fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec {
- let mut out = Vec::new();
- for (cid_wire, _filesize, tsize) in children {
- let mut link_bytes = Vec::new();
- write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash
- write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize
- write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first
- }
- write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last
- out
-}
-
-/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the
-/// multibase `b` prefix.
-pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String {
- const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
- let mut out = String::with_capacity(input.len().div_ceil(5) * 8);
- let mut buffer: u64 = 0;
- let mut bits: u32 = 0;
- for &b in input {
- buffer = (buffer << 8) | b as u64;
- bits += 8;
- while bits >= 5 {
- bits -= 5;
- let idx = ((buffer >> bits) & 0x1F) as usize;
- out.push(ALPHABET[idx] as char);
- }
- }
- if bits > 0 {
- let idx = ((buffer << (5 - bits)) & 0x1F) as usize;
- out.push(ALPHABET[idx] as char);
- }
- out
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn midhash256_matches_ts_fixtures() {
- assert_eq!(
- compute_midhash256(b"hello world"),
- "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
- );
- assert_eq!(
- compute_midhash256(b""),
- "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y"
- );
- let mb_zeros = vec![0u8; 1024 * 1024];
- assert_eq!(
- compute_midhash256(&mb_zeros),
- "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4"
- );
- }
-
- #[test]
- fn midhash256_from_sample_matches_full_bytes() {
- let small: &[u8] = b"hello world";
- assert_eq!(
- compute_midhash256_from_sample(small.len() as u64, small),
- compute_midhash256(small),
- );
-
- assert_eq!(
- compute_midhash256_from_sample(0, b""),
- compute_midhash256(b""),
- );
-
- let one_mb = vec![0u8; SAMPLE_SIZE];
- assert_eq!(
- compute_midhash256_from_sample(one_mb.len() as u64, &one_mb),
- compute_midhash256(&one_mb),
- );
-
- let two_mb = vec![0u8; 2 * SAMPLE_SIZE];
- let start = (two_mb.len() - SAMPLE_SIZE) / 2;
- let middle = &two_mb[start..start + SAMPLE_SIZE];
- assert_eq!(
- compute_midhash256_from_sample(two_mb.len() as u64, middle),
- compute_midhash256(&two_mb),
- );
- }
-
- #[test]
- fn midhash256_from_sample_partial_sample_stable() {
- let sample = vec![0u8; SAMPLE_SIZE];
- let total_5gb: u64 = 5 * 1024 * 1024 * 1024;
- let total_10gb: u64 = 10 * 1024 * 1024 * 1024;
-
- let h1 = compute_midhash256_from_sample(total_5gb, &sample);
- let h2 = compute_midhash256_from_sample(total_5gb, &sample);
- assert_eq!(h1, h2, "same (size, sample) must hash identically");
-
- let h3 = compute_midhash256_from_sample(total_10gb, &sample);
- assert_ne!(
- h1, h3,
- "size prefix must contribute to cid — same sample, different size"
- );
- }
-
- #[test]
- fn bt_v1_file_cid_wire_layout() {
- let infohash: [u8; 20] = [
- 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c,
- 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06,
- ];
-
- let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15];
- want.extend_from_slice(&infohash);
- want.push(0x04);
- assert_eq!(
- compute_bt_v1_file_cid(&infohash, 4),
- format!("b{}", base32_lower_no_padding(&want))
- );
-
- let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15];
- want0.extend_from_slice(&infohash);
- want0.push(0x00);
- assert_eq!(
- compute_bt_v1_file_cid(&infohash, 0),
- format!("b{}", base32_lower_no_padding(&want0))
- );
-
- let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16];
- want200.extend_from_slice(&infohash);
- want200.extend_from_slice(&[0xC8, 0x01]);
- assert_eq!(
- compute_bt_v1_file_cid(&infohash, 200),
- format!("b{}", base32_lower_no_padding(&want200))
- );
- }
-
- #[test]
- fn bt_v2_file_cid_wire_layout() {
- let infohash: [u8; 32] = [0xAB; 32];
- let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21];
- want.extend_from_slice(&infohash);
- want.push(0x00);
- assert_eq!(
- compute_bt_v2_file_cid(&infohash, 0),
- format!("b{}", base32_lower_no_padding(&want))
- );
- }
-
- #[test]
- fn bt_file_cid_properties() {
- let ih1 = [0x11u8; 20];
- let ih2 = [0x22u8; 32];
-
- assert_eq!(
- compute_bt_v1_file_cid(&ih1, 3),
- compute_bt_v1_file_cid(&ih1, 3)
- );
- assert_ne!(
- compute_bt_v1_file_cid(&ih1, 0),
- compute_bt_v1_file_cid(&ih1, 1)
- );
- assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b'));
- assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b'));
- assert_ne!(
- compute_bt_v1_file_cid(&[0xAB; 20], 0),
- compute_bt_v2_file_cid(&[0xAB; 32], 0)
- );
- }
-
- #[test]
- fn base32_known_vectors() {
- assert_eq!(base32_lower_no_padding(b""), "");
- assert_eq!(base32_lower_no_padding(b"f"), "my");
- assert_eq!(base32_lower_no_padding(b"fo"), "mzxq");
- assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6");
- assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq");
- assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi");
- }
-
- #[test]
- fn yt_video_cid_matches_golden_vector() {
- // Pinned by `/cid-rank-vectors.json`'s `yt-video-locator` entry. The id
- // is the real resolver output for Daft Punk / "Get Lucky" measured in
- // the delegated-playback study §4.3.
- assert_eq!(
- compute_yt_video_cid("video", "4D7u5KF7SP8").unwrap(),
- "bagecaaarav3gszdfn42ein3vgvfumn2tka4a"
- );
- }
-
- #[test]
- fn yt_video_cid_framing_matches_card_locator() {
- // The two codecs share the `varint(len(kind)) || kind || id` framing on
- // purpose, so every decoder extends a parser it already wrote. If this
- // ever fails, the two shapes have drifted and §7.2 of docs/cid-formats.md
- // is no longer true.
- let yt = compute_yt_video_cid("tmdb", "tv:95479").unwrap();
- let card = compute_card_cid("tmdb", "tv:95479").unwrap();
- // Same digest, different codec => same length, differing only in the
- // codec varint region.
- assert_ne!(yt, card);
- assert_eq!(yt.len(), card.len());
- }
-
- #[test]
- fn yt_video_cid_is_a_pure_function_of_kind_and_id() {
- // Convergence is the reason this is a codec and not a URL: two peers
- // that resolve the same video must derive the same address offline.
- assert_eq!(
- compute_yt_video_cid("video", "4D7u5KF7SP8"),
- compute_yt_video_cid("video", "4D7u5KF7SP8")
- );
- // ...and the kind is part of the identity, not decoration.
- assert_ne!(
- compute_yt_video_cid("video", "OLAK5uy_kZ8Xq"),
- compute_yt_video_cid("playlist", "OLAK5uy_kZ8Xq")
- );
- }
-
- #[test]
- fn yt_video_cid_rejects_empty_fields() {
- assert!(compute_yt_video_cid("", "4D7u5KF7SP8").is_none());
- assert!(compute_yt_video_cid("video", "").is_none());
- }
-
- #[test]
- fn ext_play_cid_matches_golden_vector() {
- assert_eq!(
- compute_ext_play_cid("https://example.bandcamp.com/album/x").unwrap(),
- "bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa"
- );
- }
-
- #[test]
- fn ext_play_cid_rejects_non_http_schemes() {
- // Load-bearing: a client hands this string to an anchor or a new tab,
- // so a `javascript:` or `data:` payload must never mint a locator.
- assert!(compute_ext_play_cid("javascript:alert(1)").is_none());
- assert!(compute_ext_play_cid("data:text/html,