From b0ff2b55c87227751c536b6317f215800120522a Mon Sep 17 00:00:00 2001 From: Worph Date: Sun, 4 Oct 2026 17:35:22 +0000 Subject: [PATCH] SDK v1.7.0 by git tag (vendored copy removed); advertises over beacon v2 as metamesh.feeder/ --- .github/workflows/publish.yml | 4 +- Cargo.lock | 5 +- Cargo.toml | 2 +- README.md | 3 +- crates/meta-feeder-sdk/Cargo.toml | 49 - crates/meta-feeder-sdk/src/cache.rs | 353 ----- crates/meta-feeder-sdk/src/common.rs | 288 ---- crates/meta-feeder-sdk/src/config.rs | 312 ----- crates/meta-feeder-sdk/src/config_page.html | 268 ---- crates/meta-feeder-sdk/src/domain.rs | 549 -------- crates/meta-feeder-sdk/src/enrich.rs | 614 --------- crates/meta-feeder-sdk/src/filename_meta.rs | 1309 ------------------- crates/meta-feeder-sdk/src/hash.rs | 1117 ---------------- crates/meta-feeder-sdk/src/lang.rs | 81 -- crates/meta-feeder-sdk/src/lib.rs | 50 - crates/meta-feeder-sdk/src/meta_core.rs | 335 ----- crates/meta-feeder-sdk/src/plugin.rs | 258 ---- crates/meta-feeder-sdk/src/query.rs | 147 --- crates/meta-feeder-sdk/src/query_eval.rs | 514 -------- crates/meta-feeder-sdk/src/serve.rs | 762 ----------- crates/meta-feeder-sdk/src/types.rs | 124 -- feeder-plugin/card-feeder/Cargo.toml | 2 +- feeder-plugin/card-feeder/Dockerfile | 5 +- 23 files changed, 10 insertions(+), 7141 deletions(-) delete mode 100644 crates/meta-feeder-sdk/Cargo.toml delete mode 100644 crates/meta-feeder-sdk/src/cache.rs delete mode 100644 crates/meta-feeder-sdk/src/common.rs delete mode 100644 crates/meta-feeder-sdk/src/config.rs delete mode 100644 crates/meta-feeder-sdk/src/config_page.html delete mode 100644 crates/meta-feeder-sdk/src/domain.rs delete mode 100644 crates/meta-feeder-sdk/src/enrich.rs delete mode 100644 crates/meta-feeder-sdk/src/filename_meta.rs delete mode 100644 crates/meta-feeder-sdk/src/hash.rs delete mode 100644 crates/meta-feeder-sdk/src/lang.rs delete mode 100644 crates/meta-feeder-sdk/src/lib.rs delete mode 100644 crates/meta-feeder-sdk/src/meta_core.rs delete mode 100644 crates/meta-feeder-sdk/src/plugin.rs delete mode 100644 crates/meta-feeder-sdk/src/query.rs delete mode 100644 crates/meta-feeder-sdk/src/query_eval.rs delete mode 100644 crates/meta-feeder-sdk/src/serve.rs delete mode 100644 crates/meta-feeder-sdk/src/types.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c9f7341..e3cd6b5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,8 +3,8 @@ name: Publish image # Build and push the feeder image to the GitHub Container Registry (ghcr.io): # * every push to main -> the moving `main` tag # * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) -# The build context is the repo root (the workspace) so the vendored -# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# The build context is the repo root (the workspace; `meta-feeder-sdk` is a +# git dependency pinned by tag); the Dockerfile lives under # feeder-plugin/card-feeder/. on: push: diff --git a/Cargo.lock b/Cargo.lock index 7381774..72870e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,7 +761,8 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "meta-feeder-sdk" -version = "0.1.0" +version = "1.7.0" +source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a" dependencies = [ "anyhow", "async-trait", @@ -770,13 +771,13 @@ dependencies = [ "bytes", "futures", "hyper", + "libc", "redb", "regex", "reqwest", "serde", "serde_json", "sha2", - "tempfile", "thiserror 1.0.69", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 97ab4aa..997ca77 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["crates/*", "feeder-plugin/*"] +members = ["feeder-plugin/*"] [workspace.package] edition = "2021" diff --git a/README.md b/README.md index e4a7f1d..dc1dbff 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ first boot; the config file wins from then on. No hot reload — bounce the feed cargo build --release --bin card-feeder cargo test -p card-feeder -# Image (build context is the repo root so the vendored SDK path dep resolves) +# Image (build context is the repo root, the workspace) docker build -f feeder-plugin/card-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-card:dev . ``` @@ -84,7 +84,6 @@ Register it with a gateway by adding it to that gateway's `gateway-config.json` ## Layout ``` -crates/meta-feeder-sdk/ vendored SDK copy (mirrored from meta-gateway) feeder-plugin/card-feeder/ src/tmdb.rs the FeederPlugin impl — routing + query + outcomes src/card.rs the Card model + its DiscoveryRecord projection diff --git a/crates/meta-feeder-sdk/Cargo.toml b/crates/meta-feeder-sdk/Cargo.toml deleted file mode 100644 index bbe98ff..0000000 --- a/crates/meta-feeder-sdk/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "meta-feeder-sdk" -version = "0.1.0" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -# Source-agnostic foundation shared by the gateway core and every feeder -# sidecar binary: the `FeederPlugin` trait, the content-addressing hash -# families, the per-plugin redb cache, the shared HTTP/cache helpers, the -# structured-query evaluator, and the axum `serve_feeders` harness that wraps -# any set of `FeederPlugin`s in the feeder HTTP contract. -# -# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes; -# the gateway core owns hashing-into-the-blockstore, the meta-core store-back, -# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT -# of here so feeders that don't need them don't pay for them. - -[dependencies] -# Async runtime + HTTP server (the serve_feeders harness) -tokio = { version = "1.40", features = ["full"] } -axum = { version = "0.7", features = ["json"] } -hyper = { version = "1", features = ["full"] } -reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } - -# Serialization -serde = { version = "1", features = ["derive"] } -serde_json = "1" -base64 = "0.22" - -# Utilities -futures = "0.3" -async-trait = "0.1" -anyhow = "1" -thiserror = "1" -tracing = "0.1" -sha2 = "0.10" -bytes = "1" -# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). -regex = "1" - -# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see -# cache.rs. Pure-Rust single-process embedded store. -redb = "2" - -[dev-dependencies] -tempfile = "3" -tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/crates/meta-feeder-sdk/src/cache.rs b/crates/meta-feeder-sdk/src/cache.rs deleted file mode 100644 index e04aae4..0000000 --- a/crates/meta-feeder-sdk/src/cache.rs +++ /dev/null @@ -1,353 +0,0 @@ -//! Per-plugin midhash cache backed by redb. -//! -//! Each plugin gets its own redb file at -//! `/gateway//cache.redb`. Keys are upstream -//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the -//! `Midhash` strings the plugin computed by fetching + hashing the upstream -//! file. -//! -//! The schema is identical to meta-share v1's so old `.redb` files copy -//! across. redb is synchronous and sub-millisecond for small ops, so calls -//! run inline from async handlers without `spawn_blocking`. - -use std::path::Path; -use std::sync::Arc; - -use redb::{Database, ReadableTableMetadata, TableDefinition}; - -pub const CACHE_FILENAME: &str = "cache.redb"; - -const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash"); -const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs"); -const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid"); -const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec"); -const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid"); -/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload -/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the -/// hash families differ: `midhash` is the synthetic midhash256-from-infohash -/// fallback; `fullhash` records "we actually downloaded the bytes and this is -/// the IPFS CIDv1 over them." -const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash"); -/// Enumerated torrent file list (JSON `Vec`) keyed by the -/// torrent's `record_id` (infohash). -const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist"); -/// Cached TMDB *search* results keyed by a stable lookup key. The value is the -/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache -/// sentinel. -const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search"); -/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`. -const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> = - TableDefinition::new("tmdb_tvdetails"); -/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`. -const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids"); -/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`. -const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> = - TableDefinition::new("tmdb_moviedetails"); -/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized -/// free-text query. -const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> = - TableDefinition::new("tmdb_principal_topn"); -/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's -/// `record_id` (infohash). Value is the JSON encoding of a `Vec`. -const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles"); -/// OpenSubtitles search results keyed by `"\x01"`. Value is the -/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel. -const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles"); -/// Small durable plugin scratch values that must survive a restart but aren't -/// content-derived — today, the torznab plugin's Prowlarr indexer-set -/// fingerprint, so a set that changed *while the feeder was down* is still -/// detected as a change on the next boot. Keys are plugin-namespaced -/// (`prowlarr:indexer-set`). redb creates the table on open, so an older -/// `.redb` file gains it transparently. -const MISC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("misc"); - -/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared -/// via `Arc`). -#[derive(Clone)] -pub struct MidhashCache { - db: Arc, -} - -/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb -/// table. -macro_rules! str_table_accessors { - ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => { - $(#[$gmeta])* - pub fn $get(&self, key: &str) -> Result, redb::Error> { - let tx = self.db.begin_read()?; - let table = tx.open_table($table)?; - Ok(table.get(key)?.map(|v| v.value().to_string())) - } - - $(#[$pmeta])* - pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> { - let tx = self.db.begin_write()?; - { - let mut table = tx.open_table($table)?; - table.insert(key, value)?; - } - tx.commit()?; - Ok(()) - } - }; -} - -impl MidhashCache { - pub fn open(cache_dir: &Path) -> Result { - let path = cache_dir.join(CACHE_FILENAME); - let db = Database::create(&path)?; - { - let tx = db.begin_write()?; - tx.open_table(MIDHASH_TABLE)?; - tx.open_table(BLOBS_TABLE)?; - tx.open_table(COVER_CID_TABLE)?; - tx.open_table(BIBREC_TABLE)?; - tx.open_table(PREVIEW_CID_TABLE)?; - tx.open_table(FULLHASH_TABLE)?; - tx.open_table(FILELIST_TABLE)?; - tx.open_table(TMDB_SEARCH_TABLE)?; - tx.open_table(TMDB_TVDETAILS_TABLE)?; - tx.open_table(TMDB_EXTIDS_TABLE)?; - tx.open_table(TMDB_MOVIEDETAILS_TABLE)?; - tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?; - tx.open_table(SUBTITLES_TABLE)?; - tx.open_table(OPENSUBTITLES_TABLE)?; - tx.open_table(MISC_TABLE)?; - tx.commit()?; - } - Ok(Self { db: Arc::new(db) }) - } - - str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE); - - str_table_accessors!( - /// Read a durable plugin scratch value (see [`MISC_TABLE`]). - get_misc, - /// Write a durable plugin scratch value (see [`MISC_TABLE`]). - put_misc, - MISC_TABLE, - ); - - pub fn entry_count(&self) -> Result { - let tx = self.db.begin_read()?; - let table = tx.open_table(MIDHASH_TABLE)?; - Ok(table.len()?) - } - - pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> { - let tx = self.db.begin_read()?; - let table = tx.open_table(BLOBS_TABLE)?; - Ok(table.get(cid)?.map(|v| v.value().to_vec())) - } - - pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> { - let tx = self.db.begin_write()?; - { - let mut table = tx.open_table(BLOBS_TABLE)?; - table.insert(cid, bytes)?; - } - tx.commit()?; - Ok(()) - } - - str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE); - - str_table_accessors!( - /// Read a previously-stored full-file IPFS cid for `record_id`. - get_fullhash, - /// Record the IPFS cid produced by a successful full fetch. - put_fullhash, - FULLHASH_TABLE - ); - - str_table_accessors!( - /// Read the cached torrent file list (JSON) for `record_id`. - get_filelist, - /// Cache the enumerated torrent file list (JSON `Vec`). - put_filelist, - FILELIST_TABLE - ); - - str_table_accessors!( - /// Read the cached subtitle linkage (JSON `Vec`). - get_subtitles, - /// Cache the discovered subtitle linkage (JSON `Vec`). - put_subtitles, - SUBTITLES_TABLE - ); - - str_table_accessors!( - /// Read a cached OpenSubtitles lookup by `"\x01"`. - get_opensubtitles, - /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel). - put_opensubtitles, - OPENSUBTITLES_TABLE - ); - - str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE); - - str_table_accessors!( - /// Read a cached TMDB search result by lookup key. - get_tmdb_search, - /// Cache a TMDB search result under `key`. - put_tmdb_search, - TMDB_SEARCH_TABLE - ); - - str_table_accessors!( - /// Read cached TMDB TV-details JSON by `tmdbid`. - get_tmdb_tvdetails, - /// Cache TMDB TV-details JSON under `tmdbid`. - put_tmdb_tvdetails, - TMDB_TVDETAILS_TABLE - ); - - str_table_accessors!( - /// Read cached TMDB external-ids JSON by `tmdbid`. - get_tmdb_extids, - /// Cache TMDB external-ids JSON under `tmdbid`. - put_tmdb_extids, - TMDB_EXTIDS_TABLE - ); - - str_table_accessors!( - /// Read cached TMDB movie-details JSON by `tmdbid`. - get_tmdb_moviedetails, - /// Cache TMDB movie-details JSON under `tmdbid`. - put_tmdb_moviedetails, - TMDB_MOVIEDETAILS_TABLE - ); - - str_table_accessors!( - /// Read the cached ranked anchor list by normalized query key. - get_tmdb_principal_topn, - /// Cache the ranked anchor list under the normalized query key. - put_tmdb_principal_topn, - TMDB_PRINCIPAL_TOPN_TABLE - ); - - pub fn get_bibrec( - &self, - record_id: &str, - ) -> Result>, redb::Error> { - let tx = self.db.begin_read()?; - let table = tx.open_table(BIBREC_TABLE)?; - let Some(v) = table.get(record_id)? else { - return Ok(None); - }; - let s = v.value(); - Ok(serde_json::from_str(s).ok()) - } - - pub fn put_bibrec( - &self, - record_id: &str, - fields: &std::collections::BTreeMap, - ) -> Result<(), redb::Error> { - let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()); - let tx = self.db.begin_write()?; - { - let mut table = tx.open_table(BIBREC_TABLE)?; - table.insert(record_id, json.as_str())?; - } - tx.commit()?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn tmp_cache_dir() -> tempfile::TempDir { - tempfile::tempdir().expect("tempdir") - } - - #[test] - fn put_then_get_returns_value() { - let dir = tmp_cache_dir(); - let cache = MidhashCache::open(dir.path()).expect("open"); - assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None); - cache.put_midhash("10.1038/x", "bafyXYZ").expect("put"); - assert_eq!( - cache.get_midhash("10.1038/x").unwrap().as_deref(), - Some("bafyXYZ") - ); - } - - #[test] - fn put_overwrites_existing_value() { - let dir = tmp_cache_dir(); - let cache = MidhashCache::open(dir.path()).expect("open"); - cache.put_midhash("k", "v1").unwrap(); - cache.put_midhash("k", "v2").unwrap(); - assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2")); - } - - #[test] - fn values_persist_across_reopens() { - let dir = tmp_cache_dir(); - { - let cache = MidhashCache::open(dir.path()).expect("open 1"); - cache.put_midhash("k", "persistent").unwrap(); - } - let cache = MidhashCache::open(dir.path()).expect("open 2"); - assert_eq!( - cache.get_midhash("k").unwrap().as_deref(), - Some("persistent") - ); - } - - #[test] - fn tmdb_search_cache_distinguishes_miss_from_absent() { - let dir = tmp_cache_dir(); - let cache = MidhashCache::open(dir.path()).expect("open"); - assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None); - cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap(); - assert_eq!( - cache - .get_tmdb_search("m\u{1}naruto\u{1}") - .unwrap() - .as_deref(), - Some("null") - ); - cache - .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}") - .unwrap(); - drop(cache); - let cache = MidhashCache::open(dir.path()).expect("reopen"); - assert_eq!( - cache - .get_tmdb_search("t\u{1}spy x family\u{1}2019") - .unwrap() - .as_deref(), - Some("{\"tmdbid\":120089}") - ); - } - - #[test] - fn tmdb_tvdetails_cache_roundtrips() { - let dir = tmp_cache_dir(); - let cache = MidhashCache::open(dir.path()).expect("open"); - assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None); - cache - .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}") - .unwrap(); - assert_eq!( - cache.get_tmdb_tvdetails("120089").unwrap().as_deref(), - Some("{\"number_of_seasons\":2}") - ); - } - - #[test] - fn entry_count_reflects_inserts() { - let dir = tmp_cache_dir(); - let cache = MidhashCache::open(dir.path()).expect("open"); - assert_eq!(cache.entry_count().unwrap(), 0); - cache.put_midhash("a", "x").unwrap(); - cache.put_midhash("b", "y").unwrap(); - assert_eq!(cache.entry_count().unwrap(), 2); - cache.put_midhash("a", "x2").unwrap(); - assert_eq!(cache.entry_count().unwrap(), 2); - } -} diff --git a/crates/meta-feeder-sdk/src/common.rs b/crates/meta-feeder-sdk/src/common.rs deleted file mode 100644 index c7a09f5..0000000 --- a/crates/meta-feeder-sdk/src/common.rs +++ /dev/null @@ -1,288 +0,0 @@ -//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`, -//! `pubmed`, `wikicommons`, `giphy`, `scihub`). -//! -//! These plugins are structurally near-identical: build a reqwest client, map -//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose -//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding -//! lives here; the genuinely-different bodies stay in the plugin modules. - -use std::path::Path; -use std::time::Duration; - -use tracing::{debug, warn}; - -use crate::cache::MidhashCache; -use crate::plugin::{ConfigError, HashKind, HashOutcome}; -use crate::types::{DiscoveryRecord, GatewayError, Hash}; - -/// Build the shared reqwest client every source plugin uses. -pub fn build_http_client( - timeout_secs: u64, - user_agent: &str, - redirect: Option, -) -> reqwest::Client { - let mut builder = reqwest::Client::builder() - .timeout(Duration::from_secs(timeout_secs)) - .user_agent(user_agent); - if let Some(policy) = redirect { - builder = builder.redirect(policy); - } - builder - .build() - .expect("reqwest client build is infallible with rustls-tls") -} - -/// Map a non-success HTTP status to a [`GatewayError`]. -/// -/// - `404` → [`GatewayError::NotFound`] -/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s) -/// - `5xx` → [`GatewayError::Transient`] -/// - anything else → [`GatewayError::Permanent`] -pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> { - let status = resp.status(); - if status.is_success() { - return Ok(()); - } - let code = status.as_u16(); - if code == 404 { - return Err(GatewayError::NotFound); - } - if code == 429 { - let retry_after = resp - .headers() - .get(reqwest::header::RETRY_AFTER) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()) - .unwrap_or(60); - return Err(GatewayError::RateLimited { - retry_after_s: retry_after, - }); - } - if status.is_server_error() { - return Err(GatewayError::Transient(format!( - "upstream {} returned {status}", - resp.url() - ))); - } - Err(GatewayError::Permanent(format!( - "upstream {} returned {status}", - resp.url() - ))) -} - -/// Open the per-plugin redb midhash cache, wrapping the error with the plugin -/// name for a clean startup diagnostic. -pub fn open_midhash_cache( - cache_dir: &Path, - plugin: &'static str, -) -> Result { - MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { - plugin, - source: anyhow::anyhow!("open redb cache: {e}"), - }) -} - -/// Cache-hit fast path for single-outcome plugins. -/// -/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record — -/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a -/// cache miss; the caller proceeds to fetch + hash. -pub fn cached_outcome( - cache: &MidhashCache, - record_id: &str, - upstream_id: &'static str, -) -> Result>, GatewayError> { - let Some(cached) = cache - .get_midhash(record_id) - .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))? - else { - return Ok(None); - }; - debug!( - target: "meta-share::gateway", - upstream = upstream_id, - record_id, - "hash cache hit" - ); - Ok(Some(vec![HashOutcome { - hash: Hash(cached), - hash_kind: HashKind::Sha2_256, - bytes: None, - record: None, - file_extension: None, - }])) -} - -/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is -/// non-fatal, so it is logged and swallowed. -pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) { - if let Err(e) = cache.put_midhash(record_id, cid) { - warn!( - target: "meta-share::gateway", - upstream = upstream_id, - record_id, - error = %e, - "hash cache put failed (non-fatal)" - ); - } -} - -/// Resolve the per-plugin cache handle, or produce the standard "not -/// configured" internal error. -pub fn require_cache<'a>( - cache: Option<&'a MidhashCache>, - plugin: &str, -) -> Result<&'a MidhashCache, GatewayError> { - cache.ok_or_else(|| { - GatewayError::Internal(anyhow::anyhow!( - "{plugin} plugin not configured (configure() never called)" - )) - }) -} - -/// Build the single-element outcome vec a single-outcome plugin returns on a -/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID, -/// and an optional file extension. -pub fn single_outcome( - cid: String, - bytes: bytes::Bytes, - record: DiscoveryRecord, - file_extension: Option, -) -> Vec { - vec![HashOutcome { - hash: Hash(cid), - hash_kind: HashKind::Sha2_256, - bytes: Some(bytes), - record: Some(record), - file_extension, - }] -} - -/// Percent-encode a query-string value using the unreserved set -/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space. -pub fn urlencode(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for b in s.as_bytes() { - match *b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { - out.push(*b as char); - } - b' ' => out.push('+'), - other => out.push_str(&format!("%{:02X}", other)), - } - } - out -} - -/// GET `url` and return the body bytes, mapping transport + status errors to -/// [`GatewayError`]. -pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result { - let resp = http - .get(url) - .send() - .await - .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; - map_status(&resp)?; - resp.bytes() - .await - .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) -} - -/// Like [`fetch_bytes`] but decodes the body as UTF-8 text. -pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result { - let resp = http - .get(url) - .send() - .await - .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; - map_status(&resp)?; - resp.text() - .await - .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) -} - -/// Decode a v1 BitTorrent infohash string into its raw 20 bytes. -/// -/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648, -/// case-insensitive). Returns `None` for anything else. -pub fn decode_infohash(s: &str) -> Option<[u8; 20]> { - let bytes = s.as_bytes(); - let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) { - let mut out = Vec::with_capacity(20); - for pair in bytes.chunks(2) { - out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?); - } - out - } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) { - decode_base32_20(bytes)? - } else { - return None; - }; - raw.try_into().ok() -} - -fn hex_nibble(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - -/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20 -/// bytes they encode. -fn decode_base32_20(chars: &[u8]) -> Option> { - let mut out = Vec::with_capacity(20); - let mut buffer: u64 = 0; - let mut bits: u32 = 0; - for &c in chars { - let val = match c { - b'A'..=b'Z' => c - b'A', - b'a'..=b'z' => c - b'a', - b'2'..=b'7' => c - b'2' + 26, - _ => return None, - } as u64; - buffer = (buffer << 5) | val; - bits += 5; - if bits >= 8 { - bits -= 8; - out.push((buffer >> bits) as u8); - } - } - Some(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn decode_infohash_hex_roundtrips() { - let hex = "0123456789abcdef0123456789abcdef01234567"; - let raw = decode_infohash(hex).expect("hex decodes"); - assert_eq!(raw[0], 0x01); - assert_eq!(raw[19], 0x67); - } - - #[test] - fn decode_infohash_is_case_insensitive_hex() { - let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd"); - let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD"); - assert!(lower.is_some()); - assert_eq!(lower, upper); - } - - #[test] - fn decode_infohash_accepts_base32() { - let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U"); - assert!(raw.is_some()); - } - - #[test] - fn decode_infohash_rejects_wrong_shape() { - assert!(decode_infohash("tooshort").is_none()); - assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none()); - assert!(decode_infohash("").is_none()); - } -} diff --git a/crates/meta-feeder-sdk/src/config.rs b/crates/meta-feeder-sdk/src/config.rs deleted file mode 100644 index c4af5d6..0000000 --- a/crates/meta-feeder-sdk/src/config.rs +++ /dev/null @@ -1,312 +0,0 @@ -//! Self-describing per-plugin configuration: schema, redaction, merge, and the -//! generic schema-driven HTML page the feeder serves at `/config`. -//! -//! The point of this module is that **the gateway carries no per-plugin config -//! knowledge** and **the SDK carries no per-plugin field names**. A plugin -//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema); -//! everything here (redaction of secrets, merge-on-save, and the rendered form) -//! is driven entirely by that schema. Add a feeder → it shows up with a working -//! config form and zero UI/gateway changes. -//! -//! ## Persistence & precedence -//! -//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to -//! `configure()`). The plugin reads **file-or-env** at startup: env is the -//! first-boot seed and stays authoritative until the operator saves through the -//! UI, at which point `config.json` exists and wins. There is no runtime reload -//! — a save returns `restart_required: true` (mirrors the gateway's -//! `gateway-config.json` posture). -//! -//! ## Secrets -//! -//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to -//! the page (redacted to a `_set: bool`), and a blank incoming value on -//! save means "keep the stored secret". - -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -/// A plugin's full config surface — an ordered list of fields. -#[derive(Serialize, Deserialize, Clone, Debug, Default)] -pub struct ConfigSchema { - pub fields: Vec, -} - -impl ConfigSchema { - /// Convenience: `true` when the plugin declares no configurable fields (the - /// page then renders a "no configuration required" notice). - pub fn is_empty(&self) -> bool { - self.fields.is_empty() - } -} - -/// One configurable field. `record_array` fields nest `fields` for the -/// per-record sub-schema (e.g. torznab's `indexers`). -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct ConfigField { - /// JSON key in the config object. - pub key: String, - /// Human label for the form. - pub label: String, - #[serde(rename = "type")] - pub kind: FieldKind, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub help: Option, - #[serde(default)] - pub required: bool, - /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated - /// as the record identifier for secret back-fill on merge (e.g. `name`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fields: Option>, -} - -impl ConfigField { - pub fn text(key: &str, label: &str) -> Self { - Self::new(key, label, FieldKind::Text) - } - pub fn secret(key: &str, label: &str) -> Self { - Self::new(key, label, FieldKind::Secret) - } - pub fn bool(key: &str, label: &str) -> Self { - Self::new(key, label, FieldKind::Bool) - } - pub fn number(key: &str, label: &str) -> Self { - Self::new(key, label, FieldKind::Number) - } - pub fn list(key: &str, label: &str) -> Self { - Self::new(key, label, FieldKind::List) - } - pub fn record_array(key: &str, label: &str, fields: Vec) -> Self { - let mut f = Self::new(key, label, FieldKind::RecordArray); - f.fields = Some(fields); - f - } - fn new(key: &str, label: &str, kind: FieldKind) -> Self { - Self { - key: key.to_string(), - label: label.to_string(), - kind, - help: None, - required: false, - fields: None, - } - } - pub fn with_help(mut self, help: &str) -> Self { - self.help = Some(help.to_string()); - self - } - pub fn required(mut self) -> Self { - self.required = true; - self - } -} - -/// Field widget / value type. Drives both rendering and the secret/merge rules. -#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum FieldKind { - /// Free text → JSON string. - Text, - /// Write-only secret → JSON string; redacted on read, "blank keeps" on save. - Secret, - /// Checkbox → JSON bool. - Bool, - /// Numeric input → JSON number. - Number, - /// Comma/line list → JSON array of strings. - List, - /// Repeatable record → JSON array of objects keyed by nested `fields`. - RecordArray, -} - -fn is_blank(v: Option<&Value>) -> bool { - match v { - None | Some(Value::Null) => true, - Some(Value::String(s)) => s.is_empty(), - _ => false, - } -} - -/// Produce a UI-safe view of `values`: secret fields are dropped and replaced -/// with a `_set` boolean; everything else passes through. Recurses into -/// `record_array` records for secret sub-fields. -pub fn redact(values: &Value, schema: &ConfigSchema) -> Value { - let src = values.as_object().cloned().unwrap_or_default(); - Value::Object(redact_object(&src, &schema.fields)) -} - -fn redact_object(src: &Map, fields: &[ConfigField]) -> Map { - let mut out = Map::new(); - for f in fields { - match f.kind { - FieldKind::Secret => { - let set = !is_blank(src.get(&f.key)); - out.insert(format!("{}_set", f.key), Value::Bool(set)); - } - FieldKind::RecordArray => { - let subs = f.fields.as_deref().unwrap_or(&[]); - let arr = src - .get(&f.key) - .and_then(Value::as_array) - .map(|recs| { - recs.iter() - .map(|r| { - let m = r.as_object().cloned().unwrap_or_default(); - Value::Object(redact_object(&m, subs)) - }) - .collect::>() - }) - .unwrap_or_default(); - out.insert(f.key.clone(), Value::Array(arr)); - } - _ => { - if let Some(v) = src.get(&f.key) { - out.insert(f.key.clone(), v.clone()); - } - } - } - } - out -} - -/// Merge `incoming` (from the UI) onto `base` (the stored/effective config), -/// schema-aware. Returns the full, UNREDACTED object to persist: -/// - secret: blank/absent incoming keeps `base`; non-blank overwrites. -/// - text/number/bool/list: present in incoming overwrites; absent keeps base. -/// - record_array: incoming array replaces base, but blank secret sub-fields are -/// back-filled from the base record matched by the first sub-field (identifier). -pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value { - let base_obj = base.as_object().cloned().unwrap_or_default(); - let inc_obj = incoming.as_object().cloned().unwrap_or_default(); - Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields)) -} - -fn merge_object( - base: &Map, - inc: &Map, - fields: &[ConfigField], -) -> Map { - let mut out = base.clone(); - for f in fields { - match f.kind { - FieldKind::Secret => { - if !is_blank(inc.get(&f.key)) { - out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap()); - } - // blank/absent → keep base (no-op) - } - FieldKind::RecordArray => { - let subs = f.fields.as_deref().unwrap_or(&[]); - let id_key = subs.first().map(|s| s.key.clone()); - let base_recs = base.get(&f.key).and_then(Value::as_array).cloned(); - let inc_recs = match inc.get(&f.key).and_then(Value::as_array) { - Some(r) => r.clone(), - None => continue, // absent → keep base array untouched - }; - let merged: Vec = inc_recs - .iter() - .map(|rec| { - let rec_obj = rec.as_object().cloned().unwrap_or_default(); - // Find the matching base record by identifier sub-field. - let base_match = match (&id_key, base_recs.as_ref()) { - (Some(idk), Some(brs)) => brs - .iter() - .find(|br| br.get(idk) == rec_obj.get(idk)) - .and_then(Value::as_object) - .cloned() - .unwrap_or_default(), - _ => Map::new(), - }; - Value::Object(merge_object(&base_match, &rec_obj, subs)) - }) - .collect(); - out.insert(f.key.clone(), Value::Array(merged)); - } - _ => { - if let Some(v) = inc.get(&f.key) { - out.insert(f.key.clone(), v.clone()); - } - } - } - } - out -} - -/// The generic, schema-driven config page served at `GET /config`. Self-contained -/// (inline CSS + JS, no external assets) so it loads identically when hit -/// directly on the feeder OR reverse-proxied through the gateway. All API calls -/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant. -pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html"); - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - fn torznab_like() -> ConfigSchema { - ConfigSchema { - fields: vec![ - ConfigField::secret("tmdb_token", "TMDB token"), - ConfigField::bool("resolve_torrent_files", "Resolve .torrent"), - ConfigField::record_array( - "indexers", - "Indexers", - vec![ - ConfigField::text("name", "Name"), - ConfigField::secret("api_key", "API key"), - ], - ), - ], - } - } - - #[test] - fn redact_hides_secrets_and_flags_set() { - let v = json!({ - "tmdb_token": "abc", - "resolve_torrent_files": true, - "indexers": [{"name":"prowlarr","api_key":"k"}] - }); - let r = redact(&v, &torznab_like()); - assert_eq!(r["tmdb_token_set"], json!(true)); - assert!(r.get("tmdb_token").is_none()); - assert_eq!(r["resolve_torrent_files"], json!(true)); - assert_eq!(r["indexers"][0]["name"], json!("prowlarr")); - assert_eq!(r["indexers"][0]["api_key_set"], json!(true)); - assert!(r["indexers"][0].get("api_key").is_none()); - } - - #[test] - fn merge_blank_secret_keeps_base() { - let base = json!({"tmdb_token": "keepme"}); - let inc = json!({"tmdb_token": ""}); - let m = merge(&base, &inc, &torznab_like()); - assert_eq!(m["tmdb_token"], json!("keepme")); - } - - #[test] - fn merge_nonblank_secret_overwrites() { - let base = json!({"tmdb_token": "old"}); - let inc = json!({"tmdb_token": "new"}); - let m = merge(&base, &inc, &torznab_like()); - assert_eq!(m["tmdb_token"], json!("new")); - } - - #[test] - fn merge_record_array_backfills_blank_secret_by_identifier() { - let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]}); - // UI re-submits the indexer with a blank api_key → keep the stored one. - let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]}); - let m = merge(&base, &inc, &torznab_like()); - assert_eq!(m["indexers"][0]["api_key"], json!("secret")); - assert_eq!(m["indexers"][0]["name"], json!("prowlarr")); - } - - #[test] - fn merge_bool_overwrites_when_present() { - let base = json!({"resolve_torrent_files": true}); - let inc = json!({"resolve_torrent_files": false}); - let m = merge(&base, &inc, &torznab_like()); - assert_eq!(m["resolve_torrent_files"], json!(false)); - } -} diff --git a/crates/meta-feeder-sdk/src/config_page.html b/crates/meta-feeder-sdk/src/config_page.html deleted file mode 100644 index 17d9d74..0000000 --- a/crates/meta-feeder-sdk/src/config_page.html +++ /dev/null @@ -1,268 +0,0 @@ - - - - - -feeder config - - - -

Plugin configuration

-

Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked * are required.

- -
- - - - - diff --git a/crates/meta-feeder-sdk/src/domain.rs b/crates/meta-feeder-sdk/src/domain.rs deleted file mode 100644 index 0e90b5e..0000000 --- a/crates/meta-feeder-sdk/src/domain.rs +++ /dev/null @@ -1,549 +0,0 @@ -//! `contentKind` → `domain` / `workForm` — the routing tables from -//! METADATA_KEYS.md §1. -//! -//! `domain` answers **"which application is this record destined for?"**. It -//! is the coarsest facet in the registry and the key every client filters its -//! catalogue on — **one value per app**, no alternation: -//! -//! ```text -//! meta-watch → domain:screen -//! meta-listen → domain:music -//! meta-read → domain:literature -//! ``` -//! -//! `workForm` answers the *other* question the old `film`/`tv` split was -//! secretly encoding: **is this work complete in itself, or one instalment of -//! a run?** (`standalone` | `serial`). It is cross-domain — a comic issue is -//! `serial` in `literature` exactly as an episode is `serial` in `screen` — -//! which is what lets a client ask for it independently of the wall. -//! -//! # Why they are stored and not derived on read -//! -//! `contentKind`'s vocabulary is deliberately open — a new feeder may mint a -//! kind and ship without a fleet-wide update. That only stays safe because the -//! domain travels *with* the record: a peer that has never heard of -//! `podcastEpisode` still routes it correctly, because it never has to -//! interpret the token. Deriving on read would silently drop every record -//! whose kind the reader predates. The same argument applies to `workForm`, -//! plus a second one: the two kinds that most need it (`pack` and `card`) are -//! exactly the two the kind alone cannot decide. -//! -//! # ⚠ The binding is permanent and one-way -//! -//! Add kinds freely; **never re-map an existing kind to a different domain**. -//! Because the value is stored, two peers on different versions that disagree -//! about where `audiobook` belongs produce a split-brain that redeploying -//! cannot heal — the wrong domain is already written into records across the -//! mesh. If a kind genuinely has to move, mint a new kind. -//! -//! The `film`+`tv` → `screen` merge (METADATA_KEYS.md §14.17) is the one -//! deliberate exception, taken once and early and paid for with a one-shot -//! meta-core corpus sweep. It is not a precedent. -//! -//! # Why `domain` and not one kind-key per medium -//! -//! `record_matches` is AND-across-keys / OR-within-key, and a non-matching key -//! fails the record outright. A per-medium split (`videoKind` / `audioKind` / -//! `documentKind`) would force meta-listen to express "music video **or** -//! audio track" as an OR across two *different* keys — which the query -//! language cannot represent at all. One domain key, OR-ed within itself, -//! always can. `workForm` is one key everywhere for the same reason. - -/// The `contentKind` → `domain` table. A kind absent from this list has no -/// domain and is **not independently routable** — correct for sidecars -/// (subtitles, artwork) and for un-anchored hits, a producer bug for anything -/// else. -/// -/// ⚠ `pack` is deliberately absent. A season pack and an album release are the -/// same structural thing, so the kind alone cannot tell `screen` from `music`; -/// the writer stamps `pack`'s domain from its own context (the same reason -/// `derive_file_type` cannot have a second `pack` arm). -/// -/// ⚠ `podcast` / `podcastEpisode` are absent too: the `spoken` domain was -/// retired unused (METADATA_KEYS.md §14.17), and the PR that first *writes* -/// those kinds picks their domain. -const DOMAIN_BY_CONTENT_KIND: &[(&str, &str)] = &[ - // screen — everything meta-watch consumes, films and serials alike. The - // standalone/serial split lives in `workForm`, not here. - ("movie", "screen"), - ("series", "screen"), - ("episode", "screen"), - // music - ("track", "music"), - ("album", "music"), - ("artist", "music"), - ("musicVideo", "music"), - ("djMix", "music"), - ("liveSet", "music"), - // literature — an audiobook's identity is the book (same author, same ISBN - // family, same series as the ebook), so meta-read groups the two editions - // on one page. The player does not decide the domain. - ("book", "literature"), - ("audiobook", "literature"), - ("comic", "literature"), - ("manga", "literature"), - ("magazine", "literature"), - // science - ("paper", "science"), -]; - -/// The `contentKind` → `workForm` table (METADATA_KEYS.md §1). -/// -/// `standalone` = a work complete in itself. `serial` = an instalment of an -/// ongoing run, **or the container or headline for one** — which is why a -/// `series` card and an `episode` are both `serial`. -/// -/// ⚠ `pack` is absent for the same reason it is absent above: a season pack is -/// `serial`, an album release is `standalone`, and only the writer knows which. -const WORK_FORM_BY_CONTENT_KIND: &[(&str, &str)] = &[ - // screen - ("movie", "standalone"), - ("series", "serial"), - ("episode", "serial"), - // music — an album is a closed work, not an ongoing run, so every rung of - // the music ladder is standalone. - ("track", "standalone"), - ("album", "standalone"), - ("artist", "standalone"), - ("musicVideo", "standalone"), - ("djMix", "standalone"), - ("liveSet", "standalone"), - // literature — the split that proves `workForm` is not a screen-only axis. - ("book", "standalone"), - ("audiobook", "standalone"), - ("comic", "serial"), - ("manga", "serial"), - ("magazine", "serial"), - // science - ("paper", "standalone"), -]; - -/// Resolve the domain a `contentKind` belongs to. -/// -/// Returns `None` for `pack` (not derivable from the kind — the writer supplies -/// it), for format/sidecar kinds that route nowhere, and for any kind this -/// build has never heard of. A caller that already knows the domain from its -/// own context must prefer that over this table. -pub fn domain_for_content_kind(content_kind: &str) -> Option<&'static str> { - lookup(DOMAIN_BY_CONTENT_KIND, content_kind) -} - -/// Resolve the work form a `contentKind` sits on. Same contract as -/// [`domain_for_content_kind`]: `None` means "not derivable", never a guess. -pub fn work_form_for_content_kind(content_kind: &str) -> Option<&'static str> { - lookup(WORK_FORM_BY_CONTENT_KIND, content_kind) -} - -fn lookup(table: &'static [(&'static str, &'static str)], key: &str) -> Option<&'static str> { - let k = key.trim(); - table - .iter() - .find(|(kind, _)| kind.eq_ignore_ascii_case(k)) - .map(|(_, value)| *value) -} - -/// Kinds whose `domain` the kind→domain table deliberately cannot decide, so -/// the **writer** supplies it (METADATA_KEYS.md §1). Today that is `pack` -/// alone: a season pack and an album release are the same structural thing, and -/// only the producer knows which — indexer-feeder's `bt.rs` reads the domain off -/// the per-file kind *before* overwriting it with `pack` precisely so the value -/// survives. -/// -/// They must ride along in every `domain:` → `contentKind:` expansion. The -/// expansion is only ever a **pre-filter** — the real `domain:` term is enforced -/// afterwards by [`crate::query_eval::record_matches`] — so widening it is free, -/// while omitting these kinds would drop exactly the records whose `domain` is -/// the most authoritative one on the wire. -pub const WRITER_SUPPLIED_DOMAIN_KINDS: &[&str] = &["pack"]; - -/// The `workForm` twin of [`WRITER_SUPPLIED_DOMAIN_KINDS`], and for the same -/// reason: a season pack is `serial`, an album release is `standalone`. -pub const WRITER_SUPPLIED_WORK_FORM_KINDS: &[&str] = &["pack"]; - -/// The inverse: every `contentKind` that belongs to `domain`. -/// -/// Used at the gateway-routing boundary. A gateway declares the *content kinds* -/// it serves (`served_content_kinds`), not domains, so a `domain:` filter is -/// matched by expanding it here rather than by adding a `served_domains` field -/// to the heartbeat — which would be a wire-protocol change requiring every -/// feeder manifest and every peer to move at once. -/// -/// ⚠ Since the `film`+`tv` merge this returns **several** kinds for `screen` -/// (`movie`, `series`, `episode`) where the two old values were disjoint. A -/// caller that reads only the first element silently means "movie" — scan the -/// whole list, or narrow with a `workForm:` term first. -pub fn content_kinds_for_domain(domain: &str) -> Vec<&'static str> { - inverse(DOMAIN_BY_CONTENT_KIND, domain) -} - -/// The inverse of the `workForm` table. Same routing-boundary role, same -/// scan-the-whole-list warning as [`content_kinds_for_domain`]. -pub fn content_kinds_for_work_form(work_form: &str) -> Vec<&'static str> { - inverse(WORK_FORM_BY_CONTENT_KIND, work_form) -} - -fn inverse(table: &'static [(&'static str, &'static str)], value: &str) -> Vec<&'static str> { - let v = value.trim(); - table - .iter() - .filter(|(_, val)| val.eq_ignore_ascii_case(v)) - .map(|(kind, _)| *kind) - .collect() -} - -/// Rewrite a query so a `domain:` / `workForm:` filter can reach a plugin that -/// has never heard of either key. -/// -/// Returns `None` when the query carries neither — the common path, and cheaper -/// than cloning. -/// -/// # Why translate instead of teaching every plugin -/// -/// `record_matches` fails a record outright when a filter key is missing from -/// its fields, and `domain` is missing from every record a not-yet-updated -/// feeder emits (`workForm` from every record written before it existed). -/// Most feeders pin the SDK by git tag, so they cannot pick up a new key on our -/// schedule. Translating `domain:literature` into the `contentKind:` -/// alternation it covers means the wire query a plugin sees never mentions -/// either key — old plugins keep working unchanged, while the dispatcher still -/// enforces the real filters afterwards, against records it has just stamped. -/// -/// # The two axes intersect -/// -/// `domain:screen workForm:serial` covers `series` and `episode` — the -/// **intersection**, not the union. Widening to the union would hand a serial -/// query the whole film corpus as a pre-filter; the post-stamp check would then -/// throw it away, having already paid for it upstream. -/// -/// An explicit `contentKind:` filter already in the query is left alone: it is -/// the narrower of the three, and the `domain`/`workForm` terms are still -/// enforced post-stamp, so the AND semantics hold either way. -pub fn expand_routing_filters( - query: &crate::query::GatewayQuery, -) -> Option { - let domains = query.filters.get("domain"); - let work_forms = query.filters.get("workForm"); - if domains.is_none() && work_forms.is_none() { - return None; - } - - let by_domain = domains.map(|vs| expand_values(vs, content_kinds_for_domain)); - let by_work_form = work_forms.map(|vs| expand_values(vs, content_kinds_for_work_form)); - - let mut out = query.clone(); - out.filters.remove("domain"); - out.filters.remove("workForm"); - if out.filters.contains_key("contentKind") { - return Some(out); - } - - let mut kinds: Vec<&'static str> = match (by_domain, by_work_form) { - (Some(d), Some(w)) => d.into_iter().filter(|k| w.contains(k)).collect(), - (Some(d), None) => d, - (None, Some(w)) => w, - (None, None) => unreachable!("guarded above"), - }; - // Domain-ambiguous kinds come along whenever the expansion is non-empty - // — see [`WRITER_SUPPLIED_DOMAIN_KINDS`]. Skipped when nothing mapped, - // so an unknown value still expands to nothing and leaves the filter - // out entirely (below) rather than narrowing to `contentKind:pack`. - if !kinds.is_empty() { - for k in WRITER_SUPPLIED_DOMAIN_KINDS - .iter() - .chain(WRITER_SUPPLIED_WORK_FORM_KINDS.iter()) - { - if !kinds.contains(k) { - kinds.push(k); - } - } - } - // An unknown value — or two axes with no kind in common — expands to - // nothing. Inserting an empty alternation would drop every record at the - // plugin; leaving the filter out lets the plugin answer and the post-stamp - // check do the (correct, empty) filtering. - if !kinds.is_empty() { - out.filters.insert( - "contentKind".to_string(), - kinds.into_iter().map(|k| k.to_string()).collect(), - ); - } - Some(out) -} - -/// Deprecated alias for [`expand_routing_filters`], kept so a caller pinned to -/// an older SDK keeps compiling. It expands `workForm:` too — the name is the -/// only thing that is stale. -#[deprecated(note = "renamed to expand_routing_filters; it now expands workForm: as well")] -pub fn expand_domain_filter( - query: &crate::query::GatewayQuery, -) -> Option { - expand_routing_filters(query) -} - -fn expand_values( - values: &[String], - expand: fn(&str) -> Vec<&'static str>, -) -> Vec<&'static str> { - let mut out: Vec<&'static str> = Vec::new(); - for v in values { - for k in expand(v) { - if !out.contains(&k) { - out.push(k); - } - } - } - out -} - -/// Stamp `domain` on a field map that already carries a `contentKind`. -/// -/// The convention from METADATA_KEYS.md §1: **whoever writes `contentKind` -/// writes `domain` in the same breath**. An existing `domain` is never -/// overwritten — a writer that knows better than the table (a `pack`, say) has -/// already set it. -pub fn stamp_domain(fields: &mut std::collections::BTreeMap) { - stamp(fields, "domain", domain_for_content_kind); -} - -/// Stamp `workForm` the same way, from the same `contentKind`. Call it -/// wherever [`stamp_domain`] is called — the registry treats the two as one -/// breath. -pub fn stamp_work_form(fields: &mut std::collections::BTreeMap) { - stamp(fields, "workForm", work_form_for_content_kind); -} - -fn stamp( - fields: &mut std::collections::BTreeMap, - key: &str, - derive: fn(&str) -> Option<&'static str>, -) { - if fields.contains_key(key) { - return; - } - let Some(kind) = fields.get("contentKind").cloned() else { - return; - }; - if let Some(value) = derive(&kind) { - fields.insert(key.to_string(), value.to_string()); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeMap; - - #[test] - fn kinds_map_to_their_domain() { - assert_eq!(domain_for_content_kind("movie"), Some("screen")); - assert_eq!(domain_for_content_kind("episode"), Some("screen")); - assert_eq!(domain_for_content_kind("track"), Some("music")); - // The case that proves medium and domain are different axes. - assert_eq!(domain_for_content_kind("musicVideo"), Some("music")); - // The case that proves the player does not decide the domain. - assert_eq!(domain_for_content_kind("audiobook"), Some("literature")); - } - - #[test] - fn film_and_tv_are_one_domain_now() { - // The merge (METADATA_KEYS.md §14.17): meta-watch's wall is one term. - assert_eq!( - domain_for_content_kind("movie"), - domain_for_content_kind("series") - ); - // ...and the split it used to carry lives here instead. - assert_eq!(work_form_for_content_kind("movie"), Some("standalone")); - assert_eq!(work_form_for_content_kind("series"), Some("serial")); - assert_eq!(work_form_for_content_kind("episode"), Some("serial")); - } - - #[test] - fn work_form_is_not_a_screen_only_axis() { - // The point of the key: it means the same thing in every domain. - assert_eq!(work_form_for_content_kind("comic"), Some("serial")); - assert_eq!(work_form_for_content_kind("book"), Some("standalone")); - assert_eq!(work_form_for_content_kind("album"), Some("standalone")); - } - - #[test] - fn the_spoken_domain_is_gone() { - // Retired unused. The kinds stay reserved with no domain until the PR - // that writes them picks one. - assert_eq!(domain_for_content_kind("podcast"), None); - assert_eq!(domain_for_content_kind("podcastEpisode"), None); - assert!(content_kinds_for_domain("spoken").is_empty()); - } - - #[test] - fn pack_has_no_derivable_domain_or_work_form() { - assert_eq!(domain_for_content_kind("pack"), None); - assert_eq!(work_form_for_content_kind("pack"), None); - } - - #[test] - fn unknown_kind_is_none_not_a_guess() { - assert_eq!(domain_for_content_kind("hologram"), None); - assert_eq!(work_form_for_content_kind("hologram"), None); - } - - #[test] - fn expansion_is_the_inverse() { - let music = content_kinds_for_domain("music"); - assert!(music.contains(&"track")); - assert!(music.contains(&"musicVideo")); - assert!(!music.contains(&"audiobook")); - for kind in content_kinds_for_domain("literature") { - assert_eq!(domain_for_content_kind(kind), Some("literature")); - } - for kind in content_kinds_for_work_form("serial") { - assert_eq!(work_form_for_content_kind(kind), Some("serial")); - } - } - - #[test] - fn screen_expands_to_every_rung_not_just_the_first() { - // The regression the merge creates for any caller reading `.first()`. - let screen = content_kinds_for_domain("screen"); - assert!(screen.contains(&"movie")); - assert!(screen.contains(&"series")); - assert!(screen.contains(&"episode")); - } - - fn q(filters: &[(&str, &[&str])]) -> crate::query::GatewayQuery { - crate::query::GatewayQuery { - raw_text: String::new(), - free_text: String::new(), - filters: filters - .iter() - .map(|(k, vs)| { - ( - k.to_string(), - vs.iter().map(|v| v.to_string()).collect::>(), - ) - }) - .collect(), - ranges: Vec::new(), - negations: Vec::new(), - } - } - - #[test] - fn no_routing_filter_is_left_untouched() { - assert!(expand_routing_filters(&q(&[("fileType", &["video"])])).is_none()); - } - - #[test] - fn domain_becomes_the_content_kinds_it_covers() { - let out = expand_routing_filters(&q(&[("domain", &["literature"])])).unwrap(); - // The plugin never sees `domain:` — that is the whole point. - assert!(!out.filters.contains_key("domain")); - let kinds = out.filters.get("contentKind").unwrap(); - assert!(kinds.contains(&"book".to_string())); - assert!(kinds.contains(&"audiobook".to_string())); - assert!(!kinds.contains(&"track".to_string())); - } - - #[test] - fn work_form_alone_also_expands() { - let out = expand_routing_filters(&q(&[("workForm", &["serial"])])).unwrap(); - assert!(!out.filters.contains_key("workForm")); - let kinds = out.filters.get("contentKind").unwrap(); - assert!(kinds.contains(&"episode".to_string())); - assert!(kinds.contains(&"comic".to_string())); - assert!(!kinds.contains(&"movie".to_string())); - } - - #[test] - fn the_two_axes_intersect_they_do_not_union() { - let out = expand_routing_filters(&q(&[ - ("domain", &["screen"][..]), - ("workForm", &["serial"][..]), - ])) - .unwrap(); - let kinds = out.filters.get("contentKind").unwrap(); - assert!(kinds.contains(&"series".to_string())); - assert!(kinds.contains(&"episode".to_string())); - // The union would have dragged the whole film corpus in. - assert!(!kinds.contains(&"movie".to_string())); - // A `serial` comic is in `literature`, not on meta-watch's wall. - assert!(!kinds.contains(&"comic".to_string())); - } - - #[test] - fn an_explicit_content_kind_filter_wins() { - let out = expand_routing_filters(&q(&[ - ("domain", &["music"][..]), - ("contentKind", &["track"][..]), - ])) - .unwrap(); - assert_eq!( - out.filters.get("contentKind").map(Vec::as_slice), - Some(&["track".to_string()][..]) - ); - } - - #[test] - fn expansion_carries_the_writer_supplied_kinds() { - // `pack` has no domain in the table (a season pack and an album are the - // same structural thing), so the writer stamps one. Leaving it out of - // the alternation would make a `domain:screen` pre-filter drop the whole - // torrent-pack corpus, which `record_matches` would then never see. - let out = expand_routing_filters(&q(&[("domain", &["screen"])])).unwrap(); - let kinds = out.filters.get("contentKind").unwrap(); - assert!(kinds.contains(&"episode".to_string())); - assert!(kinds.contains(&"pack".to_string())); - // ...and it rides along on the `workForm` axis too, exactly once. - let out = expand_routing_filters(&q(&[("workForm", &["serial"])])).unwrap(); - let kinds = out.filters.get("contentKind").unwrap(); - assert_eq!(kinds.iter().filter(|k| *k == "pack").count(), 1); - } - - #[test] - fn unknown_value_adds_no_alternation() { - // An empty alternation would drop every record at the plugin; the - // post-stamp check does the (correct, empty) filtering. - let out = expand_routing_filters(&q(&[("domain", &["hologram"])])).unwrap(); - // Not even `pack`: a lone `contentKind:pack` would be a NARROWING of a - // filter that should widen to nothing. - assert!(!out.filters.contains_key("contentKind")); - } - - #[test] - fn two_axes_with_nothing_in_common_widen_rather_than_narrow() { - // No music kind is `serial`. The pre-filter drops out; the post-stamp - // check still enforces both terms. - let out = expand_routing_filters(&q(&[ - ("domain", &["music"][..]), - ("workForm", &["serial"][..]), - ])) - .unwrap(); - assert!(!out.filters.contains_key("contentKind")); - } - - #[test] - fn stamp_fills_in_and_never_overwrites() { - let mut f: BTreeMap = BTreeMap::new(); - f.insert("contentKind".into(), "episode".into()); - stamp_domain(&mut f); - stamp_work_form(&mut f); - assert_eq!(f.get("domain").map(String::as_str), Some("screen")); - assert_eq!(f.get("workForm").map(String::as_str), Some("serial")); - - // A pack writer that already knows its context keeps it. - let mut p: BTreeMap = BTreeMap::new(); - p.insert("contentKind".into(), "pack".into()); - p.insert("domain".into(), "music".into()); - p.insert("workForm".into(), "standalone".into()); - stamp_domain(&mut p); - stamp_work_form(&mut p); - assert_eq!(p.get("domain").map(String::as_str), Some("music")); - assert_eq!(p.get("workForm").map(String::as_str), Some("standalone")); - - // No contentKind → nothing invented. - let mut n: BTreeMap = BTreeMap::new(); - stamp_domain(&mut n); - stamp_work_form(&mut n); - assert!(n.get("domain").is_none()); - assert!(n.get("workForm").is_none()); - } -} diff --git a/crates/meta-feeder-sdk/src/enrich.rs b/crates/meta-feeder-sdk/src/enrich.rs deleted file mode 100644 index b55a9f8..0000000 --- a/crates/meta-feeder-sdk/src/enrich.rs +++ /dev/null @@ -1,614 +0,0 @@ -//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline. -//! -//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base -//! record, then drives the **reused meta-sort enrichment plugins** -//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each -//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to -//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches -//! enrichment — full separation of concerns. -//! -//! ## Ordering — why we poll -//! -//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from -//! the record), and those are produced by filename-parser. So the pipeline is -//! **filename-parser → tmdb**. The plugin `/process` contract is *async* -//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after -//! kicking filename-parser we **poll meta-core** until `originalTitle` appears -//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps -//! the feeder stateless — no callback-correlation bookkeeping. The plugins' -//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`. -//! -//! The whole pipeline runs in a spawned background task so it never adds latency -//! to the libp2p resolve path; the meta-share ingester propagates the enriched -//! record on its next poll (~eventually consistent). - -use std::collections::BTreeMap; -use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; - -use serde::Serialize; -use tracing::{debug, warn}; - -use crate::meta_core::{self, Provenance}; -use crate::types::DiscoveryRecord; - -/// Operator-supplied enrichment wiring, parsed once from the feeder env. -#[derive(Clone, Debug)] -pub struct EnrichmentConfig { - /// meta-core root the feeder self-publishes to + hands to the plugins. - pub meta_core_url: String, - /// `metamesh-plugin-filename-parser` base URL (optional). - pub filename_parser_url: Option, - /// `metamesh-plugin-tmdb` base URL (optional). - pub tmdb_url: Option, - /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`. - pub tmdb_token: Option, - /// TMDB metadata language (default `en-US`). - pub tmdb_language: String, - /// Stable per-deployment id stamped into `provenance.gatewayPeer`. - pub gateway_peer: String, - /// URL the plugins POST their completion callback to (the feeder's own - /// no-op `/enrich/callback`). Polling is the real completion signal; this - /// only keeps the plugins from logging a failed-callback warning. - pub callback_url: String, - /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the - /// pipeline drives subtitle fetching after tmdb resolves the ids — the - /// single reference implementation of OpenSubtitles fetching (the gateway - /// no longer carries its own). - pub opensubtitles_url: Option, - /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`. - pub opensubtitles_api_key: Option, - /// OpenSubtitles account username (required by the plugin for downloads). - pub opensubtitles_username: Option, - /// OpenSubtitles account password. - pub opensubtitles_password: Option, - /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). - pub opensubtitles_languages: String, -} - -fn env_nonempty(key: &str) -> Option { - std::env::var(key) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the -/// app-compose convention `-feeder` → `-:8080` -/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets -/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit -/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns -/// `None` when `HOSTNAME` is unset or not in `-feeder` form, in which case -/// the caller falls back to the env seed (then soft-skips that plugin). -fn derive_sidecar_url(plugin: &str) -> Option { - let host = env_nonempty("HOSTNAME")?; - let prefix = host.strip_suffix("-feeder")?; - Some(format!("http://{prefix}-{plugin}:8080")) -} - -impl EnrichmentConfig { - /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset — - /// the **bundled-feeder** path (records returned over HTTP, the gateway - /// dispatcher stores them). Self-publishing feeders that source meta-core from - /// their own dashboard config use [`from_meta_core`](Self::from_meta_core). - pub fn from_env() -> Option { - Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?)) - } - - /// Build with an explicit meta-core root — sourced from the feeder's own - /// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The - /// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the - /// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest - /// keep their env seeds / defaults. Secrets (tmdb token, language) are - /// overlaid by the caller from the per-feeder config. - pub fn from_meta_core(meta_core_url: String) -> Self { - EnrichmentConfig { - meta_core_url, - filename_parser_url: env_nonempty("FILENAME_PARSER_URL") - .or_else(|| derive_sidecar_url("filename-parser")), - tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")), - tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")), - tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()), - gateway_peer: env_nonempty("META_GATEWAY_PEER_ID") - .or_else(|| env_nonempty("HOSTNAME")) - .unwrap_or_else(|| "gateway-feeder".to_string()), - callback_url: env_nonempty("META_FEEDER_CALLBACK_URL") - .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()), - opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"), - opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"), - opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"), - opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), - opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") - .unwrap_or_else(|| "en".to_string()), - } - } -} - -/// How long to wait for filename-parser to merge `originalTitle` before handing -/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous -/// ceiling, not an expected wait. -const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5); -/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the -/// sibling files. tmdb hits the network, so a longer ceiling than filename. -const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20); -const POLL_INTERVAL: Duration = Duration::from_millis(200); - -/// Show-level TMDB fields that are **fanned** from the primary file to every -/// sibling file in a multi-file (season-pack) torrent — they're identical for -/// every episode of a show, so we resolve tmdb **once** and copy these across -/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which -/// gateway BT records don't carry, so without this each file would re-hit TMDB). -/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`, -/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file. -const ANCHOR_EXACT: &[&str] = &[ - "tmdbid", - "imdbid", - "poster", - "backdrop", - "rating", - "releasedate", - "studio", - "genres", - "tags", -]; -const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"]; - -/// One file (or pack) to enrich: its CID, the filename to parse, and the base -/// fields already written to meta-core (handed to the plugins as `existingMeta`). -#[derive(Clone, Debug)] -pub struct EnrichTarget { - pub cid: String, - pub file_name: String, - pub base: BTreeMap, -} - -/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to -/// clone (just a reqwest client + an `Arc`), so it can be moved into a -/// spawned per-resolve task. -#[derive(Clone)] -pub struct Enricher { - http: reqwest::Client, - cfg: Arc, -} - -#[derive(Serialize)] -struct ProcessRequest<'a> { - #[serde(rename = "taskId")] - task_id: String, - cid: &'a str, - #[serde(rename = "filePath")] - file_path: &'a str, - #[serde(rename = "callbackUrl")] - callback_url: &'a str, - #[serde(rename = "metaCoreUrl")] - meta_core_url: &'a str, - #[serde(rename = "existingMeta")] - existing_meta: &'a BTreeMap, -} - -impl Enricher { - pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self { - Enricher { - http, - cfg: Arc::new(cfg), - } - } - - pub fn meta_core_url(&self) -> &str { - &self.cfg.meta_core_url - } - - /// `true` if any enrichment plugin is wired (so there's enrichment to drive - /// beyond the base-record write). - pub fn has_plugins(&self) -> bool { - self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some() - } - - /// Write the feeder's base record into meta-core (merge/upsert) under `cid`. - /// Identical shape to the gateway dispatcher's write (provenance + the - /// `cids/` key-set collapse). - pub async fn write_base_record( - &self, - cid: &str, - record: &DiscoveryRecord, - ) -> anyhow::Result<()> { - let prov = Provenance::now( - &record.upstream_id, - &record.record_id, - &self.cfg.gateway_peer, - ); - let body = meta_core::build_metadata_body(record, &prov); - meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await - } - - /// Run the video enrichment pipeline for one resolved CID: filename-parser - /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each - /// plugin writes its own fields to meta-core. Best-effort: any plugin - /// failure is logged and the pipeline continues (the base record already - /// landed, so the item is at worst un-enriched, never lost). - pub async fn enrich_video( - &self, - cid: &str, - file_name: &str, - base_fields: BTreeMap, - ) { - let mut meta = base_fields; - - // 1. filename-parser → originalTitle / movieYear / videoType / season / episode. - if let Some(fp_url) = self.cfg.filename_parser_url.clone() { - match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await { - Ok(()) => { - // Poll for the merged fields so tmdb gets the clean title. - if let Some(merged) = self - .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) - .await - { - meta = merged; - } else { - debug!( - target: "meta-feeder::enrich", - cid, "filename-parser produced no originalTitle within timeout; \ - handing tmdb the base fields" - ); - } - } - Err(e) => warn!( - target: "meta-feeder::enrich", - cid, error = %e, "filename-parser /process failed (non-fatal)" - ), - } - } - - // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video + - // a clean originalTitle, both now present on `meta`). - if let Some(tmdb_url) = self.cfg.tmdb_url.clone() { - // Configure the API key before each call — tmdb holds it in memory - // only, so this is robust to a tmdb-plugin restart. Cheap (local). - if let Err(e) = self.configure_tmdb(&tmdb_url).await { - warn!( - target: "meta-feeder::enrich", - cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)" - ); - } - if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await { - warn!( - target: "meta-feeder::enrich", - cid, error = %e, "tmdb /process failed (non-fatal)" - ); - } - } - - // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which - // tmdb writes async). Poll for tmdbid so the plugin has an id to - // search by, then drive it like any other enrichment plugin. - if self.cfg.opensubtitles_url.is_some() { - let merged = self - .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT) - .await - .unwrap_or(meta); - self.enrich_subtitles(cid, file_name, &merged).await; - } - } - - /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files + - /// an optional whole-torrent pack record. filename-parser runs on every - /// target; tmdb runs **once** on the primary video file and its show-level - /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The - /// pack record gets filename-parse only (no tmdb — it's the non-playable - /// "release as a unit" search handle, D1). Best-effort throughout. - pub async fn enrich_bundle(&self, videos: Vec, pack: Option) { - // Pack: filename-parse only, no tmdb. - if let Some(p) = pack { - self.fire_filename(&p).await; - } - - if videos.is_empty() { - return; - } - if videos.len() == 1 { - // Degenerate single-video bundle → the plain pipeline (filename → tmdb). - let v = &videos[0]; - self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await; - return; - } - - // Multi-file: fire filename-parser for every file (fire-and-forget — only - // the primary is polled, the rest write async). - for v in &videos { - self.fire_filename(v).await; - } - - // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen). - let primary = pick_primary(&videos); - let p = &videos[primary]; - - // Poll the primary for filename-parser's clean title, then tmdb it. - let merged = self - .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) - .await - .unwrap_or_else(|| p.base.clone()); - - let Some(tmdb_url) = self.cfg.tmdb_url.clone() else { - return; // no tmdb wired → filename-only enrichment, done. - }; - if let Err(e) = self.configure_tmdb(&tmdb_url).await { - warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed"); - } - if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await { - warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed"); - return; - } - - // Read the resolved anchor back off the primary, then fan it to siblings. - let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await - else { - debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan"); - return; - }; - let anchor = extract_anchor(&anchor_rec); - if anchor.is_empty() { - return; - } - for (i, v) in videos.iter().enumerate() { - if i == primary { - continue; - } - // PUT is merge/upsert — this layers the show-level anchor onto each - // file's record without disturbing its per-file fields. - if let Err(e) = - meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await - { - warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); - } - } - - // Subtitles per video: each episode needs its own (its season/episode - // differs), but they share the fanned tmdbid. Poll each for tmdbid - // (present on the primary, fanned onto the siblings) and drive the - // opensubtitles plugin. Best-effort + quota-guarded inside the plugin. - if self.cfg.opensubtitles_url.is_some() { - for v in &videos { - let merged = self - .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT) - .await - .unwrap_or_else(|| v.base.clone()); - self.enrich_subtitles(&v.cid, &v.file_name, &merged).await; - } - } - } - - /// Fire filename-parser `/process` for one target (fire-and-forget; the - /// plugin merges its fields to meta-core async). - async fn fire_filename(&self, t: &EnrichTarget) { - if let Some(fp_url) = self.cfg.filename_parser_url.clone() { - if let Err(e) = self - .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base) - .await - { - warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed"); - } - } - } - - /// POST `/process` to a plugin. Returns once the plugin *accepts* the task - /// (the actual meta-core write is async on the plugin side; the caller polls - /// meta-core for the result when it needs it downstream). - async fn process( - &self, - plugin_url: &str, - plugin: &str, - cid: &str, - file_path: &str, - existing_meta: &BTreeMap, - ) -> anyhow::Result<()> { - let url = format!("{}/process", plugin_url.trim_end_matches('/')); - let body = ProcessRequest { - task_id: format!("{plugin}-{cid}-{}", now_nanos()), - cid, - file_path, - callback_url: &self.cfg.callback_url, - meta_core_url: &self.cfg.meta_core_url, - existing_meta, - }; - let resp = self.http.post(&url).json(&body).send().await?; - if !resp.status().is_success() { - let status = resp.status(); - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("{plugin} POST {url} returned {status}: {text}"); - } - Ok(()) - } - - /// POST `/configure` to the tmdb plugin with the API key + language. - async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> { - let Some(token) = self.cfg.tmdb_token.clone() else { - anyhow::bail!("no TMDB token configured"); - }; - let url = format!("{}/configure", tmdb_url.trim_end_matches('/')); - let body = serde_json::json!({ - "config": { "apiKey": token, "language": self.cfg.tmdb_language } - }); - let resp = self.http.post(&url).json(&body).send().await?; - if !resp.status().is_success() { - let status = resp.status(); - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("tmdb /configure returned {status}: {text}"); - } - Ok(()) - } - - /// POST `/configure` to the opensubtitles plugin with the API key, account - /// credentials, and wanted languages. - async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> { - let Some(key) = self.cfg.opensubtitles_api_key.clone() else { - anyhow::bail!("no OpenSubtitles API key configured"); - }; - let cfg_url = format!("{}/configure", url.trim_end_matches('/')); - let body = serde_json::json!({ - "config": { - "apiKey": key, - "username": self.cfg.opensubtitles_username, - "password": self.cfg.opensubtitles_password, - "languages": self.cfg.opensubtitles_languages, - } - }); - let resp = self.http.post(&cfg_url).json(&body).send().await?; - if !resp.status().is_success() { - let status = resp.status(); - let text = resp.text().await.unwrap_or_default(); - anyhow::bail!("opensubtitles /configure returned {status}: {text}"); - } - Ok(()) - } - - /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or - /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta` - /// and writes the `subtitles//` key-set straight to meta-core). - /// Best-effort: any failure is logged and swallowed. - async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) { - let Some(url) = self.cfg.opensubtitles_url.clone() else { - return; - }; - if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") { - debug!( - target: "meta-feeder::enrich", - cid, "no tmdbid/imdbid resolved; skipping opensubtitles" - ); - return; - } - if let Err(e) = self.configure_opensubtitles(&url).await { - warn!( - target: "meta-feeder::enrich", - cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)" - ); - } - if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await { - warn!( - target: "meta-feeder::enrich", - cid, error = %e, "opensubtitles /process failed (non-fatal)" - ); - } - } - - /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout` - /// elapses; returns the merged field map on success. - async fn poll_until_field( - &self, - cid: &str, - field: &str, - timeout: Duration, - ) -> Option> { - let deadline = SystemTime::now() + timeout; - loop { - match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await { - Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => { - return Some(fields); - } - Ok(_) => {} - Err(e) => debug!( - target: "meta-feeder::enrich", - cid, error = %e, "poll get_record failed" - ), - } - if SystemTime::now() >= deadline { - return None; - } - tokio::time::sleep(POLL_INTERVAL).await; - } - } -} - -fn now_nanos() -> u128 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0) -} - -/// Pick the primary video file index: largest by `sizeByte`, ties broken toward -/// the first (lowest index). Falls back to 0 for an all-unsized bundle. -fn pick_primary(videos: &[EnrichTarget]) -> usize { - videos - .iter() - .enumerate() - .max_by_key(|(i, v)| { - let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0); - // larger size wins; on a tie the smaller index wins (Reverse). - (size, std::cmp::Reverse(*i)) - }) - .map(|(i, _)| i) - .unwrap_or(0) -} - -/// Project a resolved record down to the show-level anchor fields that are safe -/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]). -fn extract_anchor(fields: &BTreeMap) -> BTreeMap { - fields - .iter() - .filter(|(k, _)| { - ANCHOR_EXACT.contains(&k.as_str()) - || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p)) - }) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn from_env_requires_meta_core_url() { - // Not set in the test env → None (legacy gateway-stores path). - // (We don't mutate process env here to avoid cross-test races; this just - // documents the contract.) - assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some()); - } - - fn tgt(cid: &str, size: u64) -> EnrichTarget { - EnrichTarget { - cid: cid.into(), - file_name: cid.into(), - base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]), - } - } - - #[test] - fn pick_primary_is_largest_then_first() { - let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)]; - // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins. - assert_eq!(pick_primary(&v), 1); - } - - #[test] - fn pick_primary_all_unsized_defaults_first() { - let v = vec![ - EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() }, - EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() }, - ]; - assert_eq!(pick_primary(&v), 0); - } - - #[test] - fn extract_anchor_keeps_show_level_drops_per_file() { - let rec = BTreeMap::from_iter([ - ("tmdbid".to_string(), "10378".to_string()), - ("imdbid".to_string(), "tt1254207".to_string()), - ("poster".to_string(), "bafkPOSTER".to_string()), - ("genres".to_string(), "Animation|Comedy".to_string()), - ("plot/eng".to_string(), "a plot".to_string()), - ("tags".to_string(), "tmdb-verified".to_string()), - // per-file — must NOT be fanned: - ("originalTitle".to_string(), "Big Buck Bunny".to_string()), - ("season".to_string(), "1".to_string()), - ("episode".to_string(), "3".to_string()), - ("fileName".to_string(), "ep3.mkv".to_string()), - ]); - let a = extract_anchor(&rec); - assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng")); - assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid")); - assert!(!a.contains_key("originalTitle"), "originalTitle is per-file"); - assert!(!a.contains_key("season") && !a.contains_key("episode")); - assert!(!a.contains_key("fileName")); - } -} diff --git a/crates/meta-feeder-sdk/src/filename_meta.rs b/crates/meta-feeder-sdk/src/filename_meta.rs deleted file mode 100644 index 8c93c11..0000000 --- a/crates/meta-feeder-sdk/src/filename_meta.rs +++ /dev/null @@ -1,1309 +0,0 @@ -//! Season / episode extraction from a torrent release title. -//! -//! This is a Rust port of `@metazla/filename-tools` -//! (`packages/filename-tool/src/lib/...`) — specifically the -//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` → -//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the -//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the -//! best season/episode heuristic we have (the meta-sort `torrent` plugin -//! already uses it), so rather than invent a second, weaker heuristic for -//! the gateway we mirror its pattern set and ordering verbatim. -//! -//! ## Differences from the TS original (deliberate, documented) -//! -//! 1. **No parent-folder fallback.** The TS extractor also reads -//! `Season N` out of the *parent directory* name. A torznab hit is a -//! single release title with no directory context, so that branch is -//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup -//! simply has nothing to read. -//! 2. **Rightmost-number fallback uses the captured digits, not the raw -//! regex match.** The TS `extractRightmostNumber` does -//! `parseFloat(fullMatch)` where the match can include a leading `.` -//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat` -//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the -//! bare number) instead, which is what the function *means* to return. -//! -//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so -//! a movie's release year can't be misread as an episode number by the -//! rightmost-number fallback. - -use std::collections::BTreeSet; -use std::sync::LazyLock; - -use regex::Regex; - -/// Structured season/episode extracted from a release title. Each field -/// is the stringified number exactly as the TS extractor would emit it -/// (`"" + n`): whole numbers render without a decimal, half-episode -/// specials keep the `.5` (e.g. `"13"`, `"13.5"`). -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct SeasonEpisode { - pub season: Option, - pub episode: Option, - /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`). - /// Set together with [`episode_end`]; mutually exclusive with the - /// single-episode [`episode`] field. - pub episode_start: Option, - /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`). - pub episode_end: Option, - /// `season * 10000 + episode` disambiguator (matches the TS - /// `increment` field). Computed for completeness; the torznab caller - /// currently only persists `season`/`episode`. - pub increment: Option, - /// True when this release is a **pack/batch** rather than a single - /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season - /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with - /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack - /// is deliberately **not** given a single `episode`. - pub is_pack: bool, - /// True when [`season`] came from an explicit season-bearing token - /// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left - /// season-less (no "assume season 1" default), so a per-file reparse uses - /// this to know it should borrow the season from the pack/parent title (see - /// [`extract_season_episode_with_context`]). - pub season_explicit: bool, -} - -impl SeasonEpisode { - /// A pack carrying a concrete inclusive episode range. - pub fn has_episode_range(&self) -> bool { - self.episode_start.is_some() && self.episode_end.is_some() - } -} - -// --------------------------------------------------------------------------- -// Pattern fragments — copied 1:1 from -// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`. -// --------------------------------------------------------------------------- - -const SW: &str = r"(?:\.|\b)"; // a dot or word boundary -const S: &str = r"(?:\.|\b|\s)"; -const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored -const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5) -const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5) -const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5) -const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99 -const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5) - -/// Season-AND-episode patterns, longest/most-specific first. Each has -/// exactly two capture groups: group 1 = season, group 2 = episode. -/// Order matters — the first to match wins (mirrors the TS array order). -static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { - [ - format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), - format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), - format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"), - format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"), - format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"), - format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"), - format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"), - format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"), - format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"), - format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"), - format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"), - format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"), - ] - .iter() - .map(|p| Regex::new(p).expect("season+episode pattern compiles")) - .collect() -}); - -/// Episode-only patterns. Each has exactly one capture group: the episode. -static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { - [ - format!(r"(?i){SW}Episode{S}*{EP4}{END}"), - format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"), - format!(r"(?i){SW}E{S}*{EP4MY}{END}"), - format!(r"(?i){SW}Ep{EP4MY}{END}"), - ] - .iter() - .map(|p| Regex::new(p).expect("episode-only pattern compiles")) - .collect() -}); - -/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number. -static SOLO_EP: LazyLock = - LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles")); - -// --------------------------------------------------------------------------- -// Episode-range / pack detection. Anime + TV batches encode a *range* of -// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season -// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so -// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a -// batch collapses to one phantom episode. We detect ranges FIRST (before -// `find_season_and_episode`) and emit `episode_start`/`episode_end` + -// `is_pack`, so the caller can stamp `contentKind=pack` instead. -// -// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`: -// - ranges use a `-`/`~` separator with TWO numbers and `end > start`; -// - the bare form requires the start digit to be preceded by a space/`_`/`-` -// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read -// as a range start — that release falls through to the single-episode path; -// - 4-digit year-like / out-of-range operands are rejected. -// --------------------------------------------------------------------------- - -/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`. -static RANGE_BRACKET: LazyLock = LazyLock::new(|| { - Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles") -}); - -/// CJK episode range: `第29-38話` / `第29~38話`. -static RANGE_CJK: LazyLock = LazyLock::new(|| { - Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles") -}); - -/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces -/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a -/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight -/// separator is the disambiguator from the very common `Show 2 - 11` / -/// `Show - 11` episode-separator convention (spaced dash → single episode, not -/// a range); spaced ranges only come through the unambiguous bracketed form. -static RANGE_BARE: LazyLock = LazyLock::new(|| { - Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)") - .expect("bare range compiles") -}); - -/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single -/// season. Matched only as a fallback when no episode range is present. -static SEASON_RANGE: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles") -}); - -/// `(start, end)` is a plausible inclusive episode range: ascending, and -/// neither operand is a 4-digit year / resolution-like number. -fn plausible_range(start: u32, end: u32) -> bool { - end > start && (1..1900).contains(&start) && end < 1900 -} - -/// Detect an episode range anywhere in the (precleaned) title. Returns the -/// first plausible `(start, end)` found, trying the unambiguous bracketed and -/// CJK forms before the bare form. -fn detect_episode_range(s: &str) -> Option<(u32, u32)> { - for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] { - for caps in re.captures_iter(s) { - let start = caps.get(1).and_then(|m| m.as_str().parse::().ok()); - let end = caps.get(2).and_then(|m| m.as_str().parse::().ok()); - if let (Some(a), Some(b)) = (start, end) { - if plausible_range(a, b) { - return Some((a, b)); - } - } - } - } - None -} - -/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix. -static SFV: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles")); - -// --------------------------------------------------------------------------- -// Keyword pre-clean — copied from -// `filename-tool/src/lib/config/KeywordsArray.ts`. -// Removed before season/episode detection so release tags like `x264`, -// `1080p`, `720` can't be mistaken for episode numbers by the fallback. -// --------------------------------------------------------------------------- - -/// Word-bounded keywords (resolution/codec/source/group tags). Order is -/// irrelevant once compiled individually, but we sort longest-first to -/// match the TS constructor (`keyword.length` desc) for identical -/// overlap behaviour. -const KEYWORDS: &[&str] = &[ - "VF", - "aXXo", - "STFR", - "SUBFORCED", - "BrRipx264", - "YIFY", - "TRUEFRENCH", - "VFF", - "ATVP", - "RenewalEX", - "BD", - "DD", - "8bits", - "8 bits", - "10bits", - "10 bits", - "part 1", - "part 2", - "part 3", - "part 4", - "part A", - "part B", - "part C", - "part D", - "AAC5", - "AAC5.1", - "1080i", - "1080p", - "2160p", - "480i", - "480p", - "4k", - "576i", - "576p", - "720", - "720i", - "720p", - "aac", - "aac4", - "ac3", - "amzn", - "apple tv+", - "avi", - "bbc", - "bd5", - "bdrip", - "blu-ray", - "bluray", - "brrip", - "cam", - "cw", - "dc", - "dcu", - "ddp5.1", - "director's cut", - "disney+", - "divx", - "divx5", - "dl", - "dsr", - "dsrip", - "dts", - "dual audio", - "dubbed", - "dvd", - "dvdivx", - "dvdr", - "dvdrip", - "dvdscr", - "dvdscreener", - "eng", - "eng sub", - "esp", - "fan edit", - "fhd", - "flv", - "fs", - "ger", - "german", - "h.264", - "h.265", - "h264", - "h265", - "hardcoded", - "hbo", - "hd", - "hddvd", - "hdr", - "hdrip", - "hdtv", - "hdtvrip", - "hevc", - "hq", - "hrhd", - "hrhdtv", - "hulu", - "imax", - "ita", - "jpn", - "ld", - "md", - "mkv", - "mp3", - "mp4", - "mpeg", - "mpg", - "mq", - "multi", - "multisubs", - "netflix", - "nf", - "nfofix", - "ntsc", - "ogg", - "ogm", - "ova", - "pal", - "pdtv", - "r3", - "r5", - "rerip", - "rsvcd", - "screener", - "sd", - "se", - "subbed", - "svcd", - "tc", - "telecine", - "telesync", - "ts", - "tv series", - "uhd", - "uhdtv", - "uhdv", - "v2", - "vcd", - "vostfr", - "web", - "web-dl", - "webcast", - "webrip", - "wmv", - "ws", - "www", - "x264", - "x265", - "xsvcd", - "xvid", - "xvidvd", - "xxx", - "bits", - "AnimeServ", -]; - -/// Substrings removed without word boundaries (the TS `substringArray`). -const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"]; - -/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first. -static KEYWORD_RES: LazyLock> = LazyLock::new(|| { - let mut kws: Vec<&str> = KEYWORDS.to_vec(); - kws.sort_by_key(|k| std::cmp::Reverse(k.len())); - kws.into_iter() - .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles")) - .collect() -}); - -/// Compiled substring regexes (no boundaries, case-insensitive), longest-first. -static SUBSTRING_RES: LazyLock> = LazyLock::new(|| { - let mut subs: Vec<&str> = SUBSTRINGS.to_vec(); - subs.sort_by_key(|s| std::cmp::Reverse(s.len())); - subs.into_iter() - .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles")) - .collect() -}); - -// --------------------------------------------------------------------------- -// Language tags — anime/torznab upstreams encode language as release-name -// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a -// structured field. `extract_languages` reads *every* tag off the raw title -// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`. -// The gateway writes each as a `languages/ = "true"` key-set member -// (METADATA_KEYS §9) that the preferred-language filter matches against. -// -// Patterns are ported from the battle-tested torrent-name parsers -// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead -// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`, -// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex` -// crate has no lookaround. Two defensive choices do most of the work: -// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They -// collide with ordinary words, names, and domain TLDs. We rely on the -// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead. -// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside -// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`. -// --------------------------------------------------------------------------- - -/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD -/// or path token can't be read as a language. Only `www.`-prefixed runs are -/// removed — a generic `host.tld` matcher would eat dotted release names like -/// `Spy.x.Family`. Runs on the raw (still-dotted) title. -static URL_STRIP: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles")); - -/// Guard phrases removed from the separator-normalized title before language -/// matching, because each contains a substring that would otherwise -/// false-positive a tag. None of these phrases is itself a language tag, so -/// removal is loss-free for detection — this is how we reproduce the upstream -/// parsers' lookbehind guards without lookaround: -/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese) -/// - `tel aviv` → guards `\btel\b` (Telugu) -/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker) -static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b") - .expect("lang guard re compiles") -}); - -/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match -/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js` -/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`), -/// because meta-share's "languages I read" preference is the source of truth -/// the meta-watch filter compares against — the enriched value must be in the -/// same space the chips use. (This deliberately diverges from `gutenberg.rs` / -/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.) -/// -/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for -/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release -/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul` -/// and `fre`: every pattern that matches contributes its code, so the result -/// is a set, not a first-wins pick. -static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| { - [ - // Multi / dual-audio → "mul". `dl` matches only after "web dl" has - // been stripped (see LANG_GUARD_STRIP). - ( - r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b", - "mul", - ), - // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…). - ( - r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b", - "fre", - ), - (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"), - (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"), - (r"\bitalian\b|\bita\b", "ita"), - (r"\bgerman\b|\bger\b|\bdeu\b", "ger"), - ( - r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b", - "spa", - ), - (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"), - (r"\bkorean\b|\bkor\b", "kor"), - ( - r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b", - "chi", - ), - (r"\brussian\b|\brus\b", "rus"), - (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"), - (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"), - (r"\bczech\b|\bcze\b|\bces\b", "cze"), - (r"\bslovak\b|\bslo\b|\bslk\b", "slo"), - (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"), - ( - r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b", - "rum", - ), - (r"\bgreek\b|\bgre\b|\bell\b", "gre"), - (r"\bukrainian\b|\bukr\b", "ukr"), - (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"), - (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"), - (r"\bnorwegian\b|\bnorsk\b", "nor"), - (r"\bdanish\b|\bdansk\b", "dan"), - (r"\bfinnish\b|\bsuomi\b", "fin"), - (r"\bturkish\b|\btur\b", "tur"), - (r"\barabic\b|\bara\b", "ara"), - (r"\bhindi\b|\bhin\b", "hin"), - (r"\btamil\b|\btam\b", "tam"), - (r"\btelugu\b|\btel\b", "tel"), - (r"\bmalayalam\b|\bmal\b", "mal"), - (r"\bvietnamese\b|\bvie\b", "vie"), - (r"\bthai\b|\btha\b", "tha"), - ] - .iter() - .map(|(p, code)| { - ( - Regex::new(&format!("(?i){p}")).expect("language tag re compiles"), - *code, - ) - }) - .collect() -}); - -/// Characters `cleanSpecialChars` replaces with a space (note: `.` and -/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07` -/// structure). -const SPECIAL_CHARS: &[char] = &[ - '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"', - '`', '~', -]; - -/// Extract season/episode from a release title. -/// -/// Faithful to `computeSeasonAndEpisodeFromFilename` → -/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath` -/// (minus the parent-folder branch — see the module doc). -pub fn extract_season_episode(title: &str) -> SeasonEpisode { - let cleaned = preclean(title); - // Range detection runs on the *raw* title (with `~` normalized to `-`), - // not the precleaned one: `preclean` replaces `~` with a space (it's a - // "special char"), which would erase a `26~37` range, and it never touches - // the `(`/`[` brackets the bracketed form anchors on. - let range_src = title.replace('~', "-"); - - // Pack detection runs first — an episode range (`(01-10)`) would otherwise - // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}` - // pattern. A pack is never given a single `episode`. - if let Some((start, end)) = detect_episode_range(&range_src) { - let season = find_pack_season(&cleaned); - return SeasonEpisode { - is_pack: true, - episode_start: Some(start.to_string()), - episode_end: Some(end.to_string()), - // An explicit season token (`S2 (01-10)`) buckets the pack; a bare - // range (`Frieren (01-28)`) leaves the season for the caller. - season: season.map(|s| s.to_string()), - season_explicit: season.is_some(), - ..Default::default() - }; - } - // Season range (`S01-S03`) — a whole-series pack, no single season/episode. - if SEASON_RANGE.is_match(&range_src) { - return SeasonEpisode { - is_pack: true, - ..Default::default() - }; - } - - let (sea, ep) = find_season_and_episode(&cleaned); - - let mut out = SeasonEpisode::default(); - // season/episode 0 are valid; only < 0 means "not found". - if sea >= 0.0 { - out.season = Some(fmt_num(sea)); - out.season_explicit = true; - } - if ep >= 0.0 { - out.episode = Some(fmt_num(ep)); - // An episode with no season is left season-LESS on purpose: a season is - // only asserted when explicitly hinted — an S-token in the title here, - // or (via extract_season_episode_with_context) the parent/pack title. - // Absolute-numbered releases ("Show - 28") thus carry an episode and no - // season, so the UI buckets them under "Episodes" rather than a - // fabricated "Season 1". `increment` encodes a season, so it is only - // emitted when the season is known. - if sea >= 0.0 { - out.increment = Some(fmt_num(sea * 10000.0 + ep)); - } - } else if sea >= 0.0 { - // Season present, no episode → a whole-season pack (`Season 3`, `S03`). - out.is_pack = true; - } - out -} - -/// Extract season/episode using a `/` style context — the -/// per-file path of an opened multi-file (pack) torrent. Parses the file's own -/// `filename` first; when the filename yields an episode but **no** season, the -/// season is recovered from the `parent` (the pack/torrent title). This -/// restores the parent-folder season fallback the TS -/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped -/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves -/// to S2E3 rather than the inherited pack-level number. -pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode { - let mut file_se = extract_season_episode(filename); - // Only borrow the parent's season for a concrete single episode whose own - // filename carries no *explicit* season token (so it is otherwise - // season-less). A per-file record is an episode, not a pack, so we don't - // propagate the parent's pack/range shape here. - if file_se.episode.is_some() && !file_se.season_explicit { - if let Some(s) = find_pack_season(&preclean(parent)) { - file_se.season = Some(s.to_string()); - file_se.season_explicit = true; - if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) { - file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep)); - } - } - } - file_se -} - -/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`, -/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to -/// recover a per-file season from the parent. Excludes the bare-number season -/// form so an episode-only title isn't mis-bucketed. -fn find_pack_season(s: &str) -> Option { - let padded = format!(" {s} "); - for re in SEASON_WORD_ONLY_PATTERNS.iter() { - if let Some(caps) = re.captures(&padded) { - let v = parse_group(&caps, 1); - if v >= 0.0 { - return Some(v as i64); - } - } - } - None -} - -/// Best-effort set of languages a release supports (audio **and** subtitles, -/// mixed), inferred from its title tags. -/// -/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`, -/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we -/// read them off the raw title here. Returns the **full set** of ISO 639-2/B -/// alpha-3 codes found (matching meta-share's UI vocabulary — see -/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is -/// empty when the title carries no recognizable tag — the caller turns that -/// into the `languages/und` sentinel. -pub fn extract_languages(title: &str) -> BTreeSet { - // 1. Strip `www.…` domain noise while the dots are still intact. - let stripped = URL_STRIP.replace_all(title, " "); - // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags - // anchor inside dotted/underscored/hyphenated release names — and so - // the guard phrases ("web dl", "shang chi") match regardless of which - // separator the original used. - let norm: String = stripped - .chars() - .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c }) - .collect(); - // 3. Remove guard phrases that would otherwise false-positive a tag. - let guarded = LANG_GUARD_STRIP.replace_all(&norm, " "); - // 4. Collect every matching code (a set — dedupes, deterministic order). - let mut out = BTreeSet::new(); - for (re, code) in LANGUAGE_TAGS.iter() { - if re.is_match(&guarded) { - out.insert((*code).to_string()); - } - } - out -} - -// --------------------------------------------------------------------------- -// Quality — the resolution *tier* of a release, normalized to a single label -// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured -// resolution, so — like season/episode and language — the release title is the -// only source. Mirrors the client-side `parseVersion()` heuristic in -// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's -// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to -// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height. -// --------------------------------------------------------------------------- - -/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH` -/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title -/// carries no recognizable resolution. -pub fn extract_quality(title: &str) -> Option { - static TIER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles") - }); - static WXH: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles")); - // Normalize `.`/`_` separators so `\b`-anchored tokens match inside - // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`). - let t = title.replace(['.', '_'], " "); - if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) { - let tok = m.as_str().to_lowercase(); - return Some( - match tok.as_str() { - "4k" => "2160p", - "1080i" => "1080p", - "720i" => "720p", - other => other, - } - .to_string(), - ); - } - if let Some(h) = WXH - .captures(&t) - .and_then(|c| c.get(1)) - .and_then(|m| m.as_str().parse::().ok()) - { - let tier = if h >= 2000 { - "2160p" - } else if h >= 1000 { - "1080p" - } else if h >= 700 { - "720p" - } else if h >= 500 { - "576p" - } else { - "480p" - }; - return Some(tier.to_string()); - } - None -} - -/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12 -/// structured search facet). Title-derived / best-effort — same provenance and -/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in -/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens -/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and -/// any stream-derived value. -pub fn extract_codec(title: &str) -> Option { - static CODEC: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b") - .expect("codec re compiles") - }); - // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored - // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`). - let t = title.replace(['.', '_'], " "); - let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase(); - let cleaned = tok.replace(' ', ""); - let norm = match cleaned.as_str() { - "x265" | "h265" | "hevc" => "hevc", - "x264" | "h264" | "avc" => "h264", - other => other, // av1, xvid, divx, vp9 - }; - Some(norm.to_string()) -} - -/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + -/// `cleanSpace`, in that order. -fn preclean(filename: &str) -> String { - let mut s = filename.to_string(); - for re in KEYWORD_RES.iter() { - s = re.replace_all(&s, "").into_owned(); - } - for re in SUBSTRING_RES.iter() { - s = re.replace_all(&s, "").into_owned(); - } - // removeSFV — first occurrence only (TS uses a non-global replace). - s = SFV.replace(&s, "").into_owned(); - // cleanSpecialChars — replace each special char with a space. - s = s - .chars() - .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) - .collect(); - // cleanSpace — collapse runs of whitespace and trim. - s.split_whitespace().collect::>().join(" ") -} - -/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0` -/// for "not found". -fn find_season_and_episode(filename: &str) -> (f64, f64) { - // Pad with spaces so word-boundary fragments anchor at the ends - // exactly as the TS code does (`" " + filename + " "`). - let padded = format!(" {filename} "); - - for re in SEASON_EPISODE_PATTERNS.iter() { - if let Some(caps) = re.captures(&padded) { - let season = parse_group(&caps, 1); - let episode = parse_group(&caps, 2); - return (season, episode); - } - } - for re in EPISODE_PATTERNS.iter() { - if let Some(caps) = re.captures(&padded) { - let episode = parse_group(&caps, 1); - return (-1.0, episode); - } - } - // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught - // here — *before* the bare rightmost-number fallback — so a whole-season - // pack reports `(season, no-episode)` instead of the fallback mis-reading - // its season digit as an episode and `extract_season_episode` then - // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03` - // bug). The TS pipeline relied on a parent-folder lookup for this case, - // which the gateway dropped (see module doc). - for re in SEASON_WORD_ONLY_PATTERNS.iter() { - if let Some(caps) = re.captures(&padded) { - let season = parse_group(&caps, 1); - if season >= 0.0 { - return (season, -1.0); - } - } - } - // Last hope: the rightmost bare number. `if (number)` in the TS is - // falsy for 0, so a lone 0 doesn't count as an episode. - if let Some(n) = rightmost_number(&padded) { - if n != 0.0 { - return (-1.0, n); - } - } - (-1.0, -1.0) -} - -fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 { - caps.get(idx) - .and_then(|m| m.as_str().parse::().ok()) - .unwrap_or(-1.0) -} - -/// `extractRightmostNumber` — the last `soloEp` match's number. -fn rightmost_number(s: &str) -> Option { - SOLO_EP - .captures_iter(s) - .filter_map(|c| c.get(1)) - .last() - .and_then(|m| m.as_str().parse::().ok()) -} - -/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64 -/// `Display` already drops the trailing `.0`, so this is a thin wrapper -/// kept for intent. -fn fmt_num(n: f64) -> String { - format!("{n}") -} - -// --------------------------------------------------------------------------- -// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` + -// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`. -// The gateway previously used a separate hand-rolled `clean_torrent_title` -// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and -// season-word/episode-range stripping — so TMDB enrichment failed on many -// release-name shapes. Reuse the already-ported building blocks instead. -// --------------------------------------------------------------------------- - -/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` / -/// `S N` / trailing bare number and everything after it. -static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { - [ - format!(r"(?i){SW}Season{S}*{SEA}{END}"), - format!(r"(?i){SW}Saison{S}*{SEA}{END}"), - format!(r"(?i){SW}S{S}*{SEA}{END}"), - format!(r"(?i){SW}{SEA}{END}"), - ] - .iter() - .map(|p| Regex::new(p).expect("season-only pattern compiles")) - .collect() -}); - -/// Season-only patterns restricted to the **keyword'd** forms (`Season N`, -/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a -/// whole-season *pack* (no episode) before the bare rightmost-number fallback -/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}` -/// entry: that one matches any lone number and would turn an episode-only -/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because -/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`. -static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { - [ - format!(r"(?i){SW}Season{S}*{SEA}{END}"), - format!(r"(?i){SW}Saison{S}*{SEA}{END}"), - format!(r"(?i){SW}S{S}*{SEA}{END}"), - ] - .iter() - .map(|p| Regex::new(p).expect("season-word-only pattern compiles")) - .collect() -}); - -/// `cleanTags` — bracketed/parenthesised/braced groups. -static TAG_RE: LazyLock = - LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles")); -/// `removeTagSeparator` — stray bracket chars. -static TAG_SEP_RE: LazyLock = - LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles")); -/// `removeYear` — a standalone 19xx/20xx. -static YEAR_RE: LazyLock = - LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles")); -/// `removeBeyondHyphen` — capture everything before the first ` - ` -/// (space-delimited) hyphen. -static BEYOND_HYPHEN_RE: LazyLock = - LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles")); -static TRAILING_HYPHEN_RE: LazyLock = - LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles")); - -fn clean_special_chars(s: &str) -> String { - s.chars() - .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) - .collect() -} - -fn remove_keywords(s: &str) -> String { - let mut out = s.to_string(); - for re in KEYWORD_RES.iter() { - out = re.replace_all(&out, "").into_owned(); - } - for re in SUBSTRING_RES.iter() { - out = re.replace_all(&out, "").into_owned(); - } - out -} - -fn remove_beyond_hyphen(s: &str) -> String { - if let Some(c) = BEYOND_HYPHEN_RE.captures(s) { - return c - .get(1) - .map(|m| m.as_str().trim().to_string()) - .unwrap_or_default(); - } - TRAILING_HYPHEN_RE.replace(s, "").trim().to_string() -} - -fn remove_season_and_episode(s: &str) -> String { - for re in SEASON_EPISODE_PATTERNS - .iter() - .chain(EPISODE_PATTERNS.iter()) - .chain(SEASON_ONLY_PATTERNS.iter()) - { - if re.is_match(s) { - return re.replace(s, "").trim().to_string(); - } - } - // removeRightmostNumber fallback. - SOLO_EP.replace(s, "").trim().to_string() -} - -/// Produce a clean show/movie title from a raw release name, faithful to -/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB -/// search — NOT a hand-rolled cleaner. -pub fn clean_title(raw: &str) -> String { - let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`) - s = remove_keywords(&s); // 172 release/quality keywords + v1..v5 - s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags - s = remove_beyond_hyphen(&s); // drop everything after the first " - " - s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets - s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear - s = SFV.replace(&s, "").into_owned(); // removeSFV - s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number - s = s.replace('.', " "); // cleanDot - let s = s.trim().trim_start_matches('-').trim_end_matches('-'); - s.split_whitespace().collect::>().join(" ") -} - -#[cfg(test)] -mod tests { - use super::*; - - fn se(title: &str) -> (Option, Option) { - let r = extract_season_episode(title); - (r.season, r.episode) - } - - #[test] - fn strict_sxxeyy() { - assert_eq!( - se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"), - (Some("1".into()), Some("2".into())) - ); - } - - #[test] - fn dotted_release_name() { - assert_eq!( - se("The.Wire.S03E07.720p.BluRay.x264"), - (Some("3".into()), Some("7".into())) - ); - } - - #[test] - fn clean_title_collapses_release_variants() { - let ct = |s: &str| clean_title(s).to_lowercase(); - // Season-word, underscore/range, dotted+EnN, and the trailing-num - // Nyaa shape all collapse to the bare show title (the cases that - // were failing TMDB enrichment). - assert_eq!( - ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), - "spy x family" - ); - assert_eq!( - ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"), - "spy x family" - ); - assert_eq!( - ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"), - "spy x family" - ); - assert_eq!( - ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"), - "spy x family" - ); - } - - #[test] - fn concise_nxnn() { - assert_eq!( - se("Some Show 1x05 720p x264"), - (Some("1".into()), Some("5".into())) - ); - } - - #[test] - fn bracketed_concise() { - assert_eq!( - se("Some Show [2x10] HDTV"), - (Some("2".into()), Some("10".into())) - ); - } - - #[test] - fn verbose_season_episode() { - assert_eq!( - se("My Show Season 2 Episode 13"), - (Some("2".into()), Some("13".into())) - ); - } - - #[test] - fn anime_trailing_number_is_season_less() { - // No season marker → episode via rightmost-number fallback, and NO - // season is fabricated (absolute numbering). SFV id stripped, brackets - // ignored. The UI buckets a season-less episode under "Episodes". - assert_eq!( - se("[SubsPlease] Some Anime - 117 [FC412C51]"), - (None, Some("117".into())) - ); - } - - #[test] - fn half_episode_special_is_preserved() { - assert_eq!( - se("Some Anime S01E13.5"), - (Some("1".into()), Some("13.5".into())) - ); - } - - #[test] - fn increment_is_season_times_10000_plus_episode() { - let r = extract_season_episode("Show S02E05"); - assert_eq!(r.increment.as_deref(), Some("20005")); - } - - #[test] - fn four_digit_year_is_not_misread_as_episode() { - // The `soloEp` fallback caps at 3 digits, so a 4-digit release - // year can't be picked up — no episode here. - let r = extract_season_episode("The Matrix 1999 1080p BluRay x264"); - assert_eq!(r.episode, None); - } - - #[test] - fn short_number_in_movie_title_is_grabbed_by_fallback() { - // A bare 1-3 digit number in a movie title DOES trip the - // rightmost-number fallback (here: "13"). This is exactly why the - // torznab caller gates season/episode on TV-ness — a movie record - // must not inherit this bogus episode. - let r = extract_season_episode("Apollo 13 1080p BluRay x264"); - assert_eq!(r.episode.as_deref(), Some("13")); - } - - #[test] - fn keyword_strip_prevents_codec_false_positive() { - // "x264" must be removed so "264" isn't read as the episode. - let r = extract_season_episode("Plain Movie Title x264"); - assert_eq!(r.episode, None); - } - - #[test] - fn no_numbers_yields_nothing() { - let r = extract_season_episode("Just A Plain Title"); - assert_eq!(r.season, None); - assert_eq!(r.episode, None); - } - - #[test] - fn season_only_pack_is_not_misread_as_episode() { - // Whole-season packs (no episode) must report the season with NO - // episode — not `S01E0N` from the rightmost-number fallback. This is - // the original bug: `Spy x Family Season 3 (S03)` was parsed as - // season 1, episode 3, dumping the S3 pack into the Season-1 bucket. - assert_eq!( - se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"), - (Some("3".into()), None) - ); - assert_eq!( - se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"), - (Some("2".into()), None) - ); - // Bare `S03` pack (no `Season` word) via the S-prefixed pattern. - assert_eq!( - se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"), - (Some("3".into()), None) - ); - assert_eq!( - se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"), - (Some("3".into()), None) - ); - // French season word. - assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None)); - } - - #[test] - fn explicit_sxxeyy_still_wins_over_season_only() { - // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected - // by the new season-only branch. - assert_eq!( - se("Spy x Family Season 3 S03E13 1080p"), - (Some("3".into()), Some("13".into())) - ); - } - - #[test] - fn episode_range_is_a_pack_not_sxxeyy() { - // The headline bug: `(01-10)` was mis-read as S1 E10 by the - // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no - // single `episode`. - let r = extract_season_episode("Frieren (01-28) [Batch] 1080p"); - assert!(r.is_pack); - assert_eq!(r.episode, None); - assert_eq!(r.episode_start.as_deref(), Some("1")); - assert_eq!(r.episode_end.as_deref(), Some("28")); - - // Bare, dash-delimited, leading-zero range. - let r = extract_season_episode("Some Anime 01-12 1080p"); - assert!(r.is_pack); - assert_eq!(r.episode_start.as_deref(), Some("1")); - assert_eq!(r.episode_end.as_deref(), Some("12")); - - // Tilde range, no leading zero. - let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]"); - assert!(r.is_pack); - assert_eq!(r.episode_start.as_deref(), Some("26")); - assert_eq!(r.episode_end.as_deref(), Some("37")); - } - - #[test] - fn explicit_season_buckets_a_pack() { - // `S2 (01-10)` → pack in season 2, episodes 1–10. - let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)"); - assert!(r.is_pack); - assert_eq!(r.season.as_deref(), Some("2")); - assert_eq!(r.episode_start.as_deref(), Some("1")); - assert_eq!(r.episode_end.as_deref(), Some("10")); - } - - #[test] - fn cjk_episode_range_is_a_pack() { - let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]"); - assert!(r.is_pack); - assert_eq!(r.episode_start.as_deref(), Some("29")); - assert_eq!(r.episode_end.as_deref(), Some("38")); - } - - #[test] - fn season_range_is_a_pack_with_no_single_season() { - let r = extract_season_episode("Frieren S01-S03 Complete 1080p"); - assert!(r.is_pack); - assert_eq!(r.season, None); - assert_eq!(r.episode, None); - assert_eq!(r.episode_start, None); - } - - #[test] - fn whole_season_release_is_a_pack() { - // A season-only release (`Season 3`, `S03`, no episode) is a pack too — - // the original "season-level release shows as one media card" bug. - let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)"); - assert!(r.is_pack); - assert_eq!(r.season.as_deref(), Some("3")); - assert_eq!(r.episode, None); - } - - #[test] - fn single_episode_is_not_a_pack() { - // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all - // single episodes, never packs. - for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] { - assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack"); - } - let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]"); - assert!(!r.is_pack); - assert_eq!(r.season.as_deref(), Some("2")); - assert_eq!(r.episode.as_deref(), Some("7")); - - // The `Show - ` anime convention (spaced dash) is a - // single episode, NOT a `2..11` range — the tight-separator rule keeps - // this out of pack detection. - let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]"); - assert!(!r.is_pack); - assert_eq!(r.season.as_deref(), Some("2")); - assert_eq!(r.episode.as_deref(), Some("11")); - - // A year range in a movie title is not an episode range. - assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack); - } - - #[test] - fn per_file_context_recovers_season_from_parent() { - // Opened pack: the inner file carries the episode, the pack title the - // season. `Frieren - 03.mkv` alone is season-less; with the S2 pack - // parent it resolves to S2E3 (the parent is the "season hint"). - let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv"); - assert_eq!(r.season.as_deref(), Some("2")); - assert_eq!(r.episode.as_deref(), Some("3")); - assert!(!r.is_pack); - - // The file's own explicit season wins over the parent. - let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv"); - assert_eq!(r.season.as_deref(), Some("1")); - assert_eq!(r.episode.as_deref(), Some("5")); - } - - #[test] - fn absolute_numbered_anime_is_season_less() { - // No season keyword → the rightmost-number fallback still finds the - // episode, but NO season is fabricated. Season is only ever asserted - // from an explicit token (title or parent), never inferred — so an - // absolute-numbered release lands in the UI's "Episodes" bucket. - assert_eq!( - se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"), - (None, Some("37".into())) - ); - } - - fn langs(s: &str) -> BTreeSet { - extract_languages(s) - } - - #[test] - fn language_from_release_tags() { - // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi). - assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre")); - assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger")); - assert!(langs("Show ENG SUB 720p").contains("eng")); - assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn")); - assert!(langs("Film TRUEFRENCH 1080p").contains("fre")); - } - - #[test] - fn multi_is_mul_and_collects_every_language() { - // `languages` is the union of all supported audio+sub languages. - // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is - // ALSO collected (the set, not a first-wins pick). - assert_eq!( - langs("Show MULTI VOSTFR 1080p"), - BTreeSet::from(["mul".to_string(), "fre".to_string()]) - ); - assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul")); - assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul")); - // A bilingual release collects both specific codes. - assert_eq!( - langs("Movie.2021.GERMAN.ENGLISH.1080p"), - BTreeSet::from(["ger".to_string(), "eng".to_string()]) - ); - } - - #[test] - fn no_language_tag_yields_empty_set() { - // A plain quality/codec release name carries no language tag — the - // caller maps the empty set to the `languages/und` sentinel. - assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty()); - // Short codes embedded in words/names must not false-positive: "ita" - // inside "digital", "por" inside "portal", etc. - assert!(langs("VFX Reel 2024 1080p").is_empty()); - assert!(langs("Digital Capital Portal 1080p").is_empty()); - } - - #[test] - fn lookaround_guards_reproduced_in_code() { - // WEB-DL must not read as `dl`→mul (German dual-language marker). - assert!(!langs("Movie 2021 WEB-DL x264").contains("mul")); - // "Shang-Chi" must not read as `chi` (Chinese)… - assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi")); - // …but a real CHI tag still does. - assert!(langs("Movie.2021.CHI.1080p").contains("chi")); - // "Tel Aviv" must not read as `tel` (Telugu). - assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel")); - // www. domain noise (incl. its TLD) must not contribute a language. - assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty()); - } - - #[test] - fn quality_tiers_and_normalization() { - let q = |s: &str| extract_quality(s); - assert_eq!( - q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(), - Some("1080p") - ); - assert_eq!( - q("The.Wire.S03E07.720p.BluRay.x264").as_deref(), - Some("720p") - ); - // 4k / interlaced collapse to the canonical progressive tier. - assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p")); - assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p")); - assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p")); - // WxH fallback, mapped by height; dot/underscore separators handled. - assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p")); - assert_eq!(q("Show 1280x720").as_deref(), Some("720p")); - // No resolution token → None (never guess). - assert_eq!(q("Just A Plain Title"), None); - } - - #[test] - fn codec_normalization() { - let c = |s: &str| extract_codec(s); - // HEVC encoder spellings collapse to the codec name. - assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc")); - assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc")); - assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc")); - // AVC encoder spellings collapse to h264. - assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264")); - assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264")); - // Others kept as-is. - assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1")); - assert_eq!(c("Old Release XviD").as_deref(), Some("xvid")); - // No codec token → None (never guess). - assert_eq!(c("Just A Plain Title 1080p"), None); - } -} diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs deleted file mode 100644 index 86a1ab2..0000000 --- a/crates/meta-feeder-sdk/src/hash.rs +++ /dev/null @@ -1,1117 +0,0 @@ -//! Content-addressed identifiers used by feeder plugins' `compute_outcomes` -//! to hash upstream-fetched bytes. -//! -//! Two function families live here: -//! -//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] — -//! the size-prefix-plus-middle-1MB-sample identification primitive shared -//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`. -//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the -//! middle slice directly so plugins that can only fetch a partial range -//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file -//! just to throw most of it away. **Rust-only by construction.** -//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full -//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec -//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches -//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 -//! --hash=sha2-256 `. -//! -//! Cross-implementation parity for `compute_midhash256` with the TypeScript -//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression -//! silently breaks dedupe across the platform. - -use bytes::Bytes; -use sha2::{Digest, Sha256}; - -/// Sample window in bytes (1 MiB). Files this size or smaller are hashed -/// in full; larger files are hashed over the middle 1 MiB slice. -const SAMPLE_SIZE: usize = 1024 * 1024; - -/// Per-block IPFS chunk size (256 KiB), matches kubo's -/// `--chunker=size-262144` default. -pub const IPFS_CHUNK_SIZE: usize = 256 * 1024; - -/// dag-pb fanout — children per internal node. Matches kubo's balanced -/// builder default. -pub const IPFS_FANOUT: usize = 174; - -/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every -/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by -/// their own CID. Pass `.blocks` to the gateway's blockstore so peers -/// can fetch the file by CID via bitswap. -#[derive(Debug, Clone)] -pub struct IpfsBlocks { - /// Canonical "bafy…" / "bafk…" root cid string. Identical to what - /// [`compute_ipfs_cid`] returns for the same input. - pub root: String, - /// Every block — root included — keyed by its own cid. For a - /// single-leaf file, this is a one-entry vec `[(root, payload)]` - /// (raw codec, payload is the file bytes). For multi-leaf files, - /// it is `leaves ++ internal_nodes` in build order; the **last** - /// entry is the root. - pub blocks: Vec<(String, Bytes)>, -} - -/// Compute a midhash256 CID matching the TypeScript implementation in -/// `meta-hash/src/lib/file-id/FastHash.ts`: -/// -/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the -/// whole file if it fits in 1 MiB). -/// 2. SHA-256 it. -/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field -/// and the multihash hash-code field, length 32, then multibase-base32-lower -/// with the `b` prefix. -/// -/// **Endianness for the size prefix is big-endian** to match TS's -/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every -/// import — keep the test pinned. -pub fn compute_midhash256(bytes: &[u8]) -> String { - let size = bytes.len(); - let sample: &[u8] = if size <= SAMPLE_SIZE { - bytes - } else { - let start = (size - SAMPLE_SIZE) / 2; - &bytes[start..start + SAMPLE_SIZE] - }; - compute_midhash256_from_sample(size as u64, sample) -} - -/// Same midhash256 primitive as [`compute_midhash256`], but takes the -/// sample bytes directly rather than slicing them out of a full file. -/// For upstreams that can only deliver a middle range (BitTorrent BEP 9, -/// sliced HTTP), this avoids forcing the caller to buffer the whole file -/// in memory just to throw most of it away. -/// -/// `total_size` is the **full file size in bytes** — used as the -/// size prefix, NOT `middle_sample.len()`. -pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String { - let size_be = total_size.to_be_bytes(); - let mut hasher = Sha256::new(); - hasher.update(size_be); - hasher.update(middle_sample); - let digest: [u8; 32] = hasher.finalize().into(); - - // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20]. - // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] - let mut wire = Vec::with_capacity(38); - wire.push(0x01); - wire.extend_from_slice(&[0x80, 0x20]); - wire.extend_from_slice(&[0x80, 0x20]); - wire.push(0x20); - wire.extend_from_slice(&digest); - - format!("b{}", base32_lower_no_padding(&wire)) -} - -/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1. -/// -/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so -/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11), -/// len=20, digest=`. -pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String { - // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] - let mut wire = Vec::with_capacity(4 + 20); - wire.push(0x01); // CIDv1 version - wire.push(0x55); // codec: raw - wire.push(0x11); // multihash: sha1 - wire.push(0x14); // multihash length: 20 - wire.extend_from_slice(infohash_20); - format!("b{}", base32_lower_no_padding(&wire)) -} - -/// Custom multicodec for a single file inside a **BitTorrent v1** torrent -/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`. -pub const BTIH_V1_FILE_CODEC: u64 = 0x1001; - -/// Custom multicodec for a single file inside a **BitTorrent v2** torrent -/// (`btih-v2-file`). -pub const BTIH_V2_FILE_CODEC: u64 = 0x1002; - -/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent, -/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info` -/// dict) plus the zero-based file index in `info.files`. Single-file -/// torrents use index `0`. -pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String { - bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index) -} - -/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent -/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's -/// canonical traversal index in `file tree`. -pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String { - bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index) -} - -/// Shared encoder for the two torrent-file CID families. `infohash` is 20 -/// bytes for v1 / 32 for v2; `codec` selects the family. -fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String { - // digest = infohash ‖ varint(file_index) - let mut digest = Vec::with_capacity(infohash.len() + 2); - digest.extend_from_slice(infohash); - write_pb_varint(file_index, &mut digest); - - // CIDv1: [version][codec varint][multihash code varint][len varint][digest] - let mut wire = Vec::with_capacity(1 + 4 + digest.len()); - wire.push(0x01); - write_pb_varint(codec, &mut wire); - write_pb_varint(codec, &mut wire); - write_pb_varint(digest.len() as u64, &mut wire); - wire.extend_from_slice(&digest); - format!("b{}", base32_lower_no_padding(&wire)) -} - -/// Custom multicodec for a self-describing **Newznab release** locator -/// (`nzb-release`). MetaMesh-private, adjacent to the torrent-file codecs in the -/// `0x10xx` range. Unlike a content hash, the cid *embeds* the indexer host + -/// release id in an identity multihash, so the credentialed meta-share peer -/// decodes `{host,id}` straight from the cid and grabs the `.nzb` via `t=get` -/// only at playback — no side-table, no `.nzb` download at search time. (The -/// `0x1004` `sha256(host‖id)` opaque hash and the `0x1003` `nzb-posting` -/// variant were both removed; only this self-describing form is emitted.) -pub const NZB_RELEASE_CODEC: u64 = 0x1005; - -/// Digest byte ceiling for an `nzb-release` cid, matching meta-share's -/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. -const NZB_RELEASE_MAX_DIGEST: usize = 64; - -/// Encode a self-describing `nzb-release` CID from a Newznab indexer's host + -/// bare release id (the hex `` id). The multihash is *identity* (`0x00`) -/// and its digest embeds the locator as `varint(host_len) ‖ host ‖ id_bytes`, -/// where `id_bytes` is the hex-decoded id. meta-share's `decode_nzb_release_cid` -/// is the exact inverse. Host-namespaced: the same release on two indexers -/// yields different cids. -/// -/// Returns `None` when the id isn't hex or the digest would overflow the -/// multihash budget (an oversized host) — the caller drops that row. -pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option { - let id_bytes = decode_hex_id(release_id)?; - let host_b = api_base.as_bytes(); - - // identity-multihash digest = varint(host_len) ‖ host ‖ id_bytes - let mut digest = Vec::with_capacity(2 + host_b.len() + id_bytes.len()); - write_pb_varint(host_b.len() as u64, &mut digest); - digest.extend_from_slice(host_b); - digest.extend_from_slice(&id_bytes); - if digest.len() > NZB_RELEASE_MAX_DIGEST { - return None; - } - - // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] - let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); - wire.push(0x01); - write_pb_varint(NZB_RELEASE_CODEC, &mut wire); - wire.push(0x00); // multihash code: identity - write_pb_varint(digest.len() as u64, &mut wire); - wire.extend_from_slice(&digest); - Some(format!("b{}", base32_lower_no_padding(&wire))) -} - -/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the -/// id is all-hex by construction; odd-length ids are left-padded with a `0` -/// nibble. Returns `None` on any non-hex byte (defensive). -fn decode_hex_id(id: &str) -> Option> { - let padded; - let s: &str = if id.len() % 2 == 1 { - padded = format!("0{id}"); - &padded - } else { - id - }; - let bytes = s.as_bytes(); - let mut out = Vec::with_capacity(bytes.len() / 2); - for pair in bytes.chunks(2) { - let hi = (pair[0] as char).to_digit(16)?; - let lo = (pair[1] as char).to_digit(16)?; - out.push(((hi << 4) | lo) as u8); - } - Some(out) -} - -/// Custom multicodec for a **card locator** — the identity of a *work* (a -/// series, a film) as published by a metadata bridge, rather than of any -/// bytes. MetaMesh-private, the next free slot after `0x1006` url. (`0x1004` -/// nzb-release-hash stays **retired — do not reuse**. `0x1003` was retired too, -/// but has since been **revived** for the self-scanned Usenet posting identity — -/// see [`NZB_POSTING_CODEC`] and `METADATA_KEYS.md` §2.) -/// -/// This is the first CID family in the platform where **no bytes exist -/// anywhere, ever** — `0x1005`/`0x1006` are locators too, but both eventually -/// resolve to bytes (NNTP, HTTP), and `0x1000` midhash256 is a digest over -/// real file bytes. A card is pure identity: the record *is* the payload, and -/// it lives in meta-core. Consequences, all already handled: -/// -/// * Bitswap seeding is gated on `HashKind::Sha2_256` (gateway invariant 6), -/// so a card is never offered to the swarm — nothing to seed. -/// * meta-share's byte path 404s for it, which is correct. -/// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for -/// `canonical_cid` — which is what makes the future "one card, two CIDs" -/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe. -pub const CARD_LOCATOR_CODEC: u64 = 0x1007; - -/// Digest byte ceiling for a card locator, matching meta-share's -/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. -const CARD_LOCATOR_MAX_DIGEST: usize = 64; - -/// Encode a self-describing **card locator** CID from a metadata source and -/// that source's own id for the work (`("tmdb", "tv:95479")`, -/// `("mal", "52991")`). The multihash is *identity* (`0x00`) and its digest -/// embeds the locator as `varint(source_len) ‖ source ‖ id`, both UTF-8 — -/// exactly the shape [`compute_nzb_release_cid`] uses, minus the hex decode -/// (a card id is arbitrary text, not a hex release id). -/// -/// **The CID is a pure function of `(source, id)`**, which is the whole point: -/// any peer that knows a tmdb id can derive the card's address offline and -/// look it up locally, with no discovery round-trip. It also means the same -/// card discovered by two different peers converges on one meta-core record -/// for free, via the `cids/` reverse index. -/// -/// Source-namespaced: the same work on TMDB and on MyAnimeList yields -/// different cids. Merging those two into one record is a meta-core alias -/// operation driven by an explicit cross-source id mapping — never inferred -/// here (see `meta-gateway/docs/others/card-tier-search.md` §9). -/// -/// Returns `None` when `source` is empty or the digest would overflow the -/// multihash budget — the caller drops that card. -pub fn compute_card_cid(source: &str, id: &str) -> Option { - if source.is_empty() || id.is_empty() { - return None; - } - let source_b = source.as_bytes(); - let id_b = id.as_bytes(); - - // identity-multihash digest = varint(source_len) ‖ source ‖ id - let mut digest = Vec::with_capacity(2 + source_b.len() + id_b.len()); - write_pb_varint(source_b.len() as u64, &mut digest); - digest.extend_from_slice(source_b); - digest.extend_from_slice(id_b); - if digest.len() > CARD_LOCATOR_MAX_DIGEST { - return None; - } - - // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] - let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); - wire.push(0x01); - write_pb_varint(CARD_LOCATOR_CODEC, &mut wire); - wire.push(0x00); // multihash code: identity - write_pb_varint(digest.len() as u64, &mut wire); - wire.extend_from_slice(&digest); - Some(format!("b{}", base32_lower_no_padding(&wire))) -} - -// --------------------------------------------------------------------------- -// Delegated-playback locators — `yt-video` (0x1008) and `ext-play` (0x1009) -// --------------------------------------------------------------------------- -// -// These two are the limit case *beyond* `card` (0x1007). `card` addresses a -// work with no bytes anywhere; these address a **rendition whose bytes exist -// but are permanently someone else's**. An external player renders the media, -// nothing is transported, nothing is content-addressed, nothing is re-seedable. -// -// See `docs/cid-formats.md` §7 and -// `docs/study/listenbrainz-youtube-playback-tier-2026-08-21.md`. - -/// Custom multicodec for a **YouTube rendition** — delegated playback. -/// -/// Its own codec rather than a `url` (`0x1006`) for three reasons, in -/// descending order of how much trouble the alternative causes: -/// -/// * **`0x1006` means "fetch these bytes once and seed them".** Point a `url` -/// locator at a `watch` page and meta-share does exactly that: fetches the -/// HTML, chunks it, seeds it, and links the resulting content CID back onto -/// the record *as the file*. That is the MSR1 poisoning shape, except -/// permanent and replicated across peers. A resolved `googlevideo` URL is no -/// better — time-limited and IP-bound, so it is not a locator at all. -/// * **The id is canonical, a URL is not.** `youtube.com` / `music.youtube.com` -/// / `youtu.be` all address the same video, and query parameters are tracking -/// noise, so the same rendition would mint several different CIDs. Being a -/// pure function of `(kind, id)`, this codec makes two peers that resolve the -/// same track converge on one meta-core record for free, via the -/// `cids/` reverse index — the property that makes [`compute_card_cid`] -/// useful, for the same reason. -/// * **It is tiny** — 11 bytes of id plus a short kind prefix. -/// -/// ⚠ **An immutable id is not durable playback.** Takedowns, geo-blocks and -/// embed-disabled uploads all kill a reference whose CID stays valid forever — -/// the same rot class as [`NZB_RELEASE_CODEC`]. The mitigation is at the -/// *record* level (carry several ranked references per recording so a client -/// can fall through), never at the CID level. -pub const YT_VIDEO_CODEC: u64 = 0x1008; - -/// Custom multicodec for an **external page to open, never fetch**. -/// -/// Wire-identical to `url` (`0x1006`) apart from the codec, and that difference -/// is the entire contract: `0x1006` resolves cache-through and lands in the -/// blockstore, `0x1009` **must be refused by the byte path** and only ever -/// surfaces as an "Open on …↗" affordance. Use it for anything worth linking -/// but not worth a per-provider codec (Bandcamp, Jamendo, an arbitrary stream -/// page). -pub const EXT_PLAY_CODEC: u64 = 0x1009; - -/// Encode a [`YT_VIDEO_CODEC`] locator from a `kind` and a bare YouTube id. -/// -/// `kind` is `"video"` | `"playlist"` | `"channel"`; `id` is the bare -/// identifier (`4D7u5KF7SP8`, `OLAK5uy_…`, `UCRr1xG_2WIDs18a6cIiCxeA`) — **not** -/// a URL, and not a `watch?v=` fragment. The framing is byte-identical to -/// [`compute_card_cid`] on purpose, so each of the seven rank/decode -/// implementations extends a parser it already wrote rather than learning a -/// novel shape. -/// -/// Returns `None` on an empty field or a digest past the multihash budget — -/// the caller drops that reference. -pub fn compute_yt_video_cid(kind: &str, id: &str) -> Option { - kinded_locator_cid(YT_VIDEO_CODEC, kind, id) -} - -/// Encode an [`EXT_PLAY_CODEC`] locator from an `http(s)` URL. -/// -/// The scheme check is not cosmetic: it is what keeps a `javascript:` or -/// `data:` payload from ever reaching a client that will hand this string to -/// an anchor or a new tab. -pub fn compute_ext_play_cid(url: &str) -> Option { - if !(url.starts_with("http://") || url.starts_with("https://")) { - return None; - } - let digest = url.as_bytes(); - if digest.is_empty() || digest.len() > CARD_LOCATOR_MAX_DIGEST { - return None; - } - Some(identity_locator_cid(EXT_PLAY_CODEC, digest)) -} - -/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used -/// by [`compute_card_cid`] and [`compute_yt_video_cid`]. -fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option { - if kind.is_empty() || id.is_empty() { - return None; - } - let kind_b = kind.as_bytes(); - let id_b = id.as_bytes(); - - let mut digest = Vec::with_capacity(2 + kind_b.len() + id_b.len()); - write_pb_varint(kind_b.len() as u64, &mut digest); - digest.extend_from_slice(kind_b); - digest.extend_from_slice(id_b); - if digest.len() > CARD_LOCATOR_MAX_DIGEST { - return None; - } - Some(identity_locator_cid(codec, &digest)) -} - -/// CIDv1 with an *identity* multihash: -/// `[version=0x01][codec varint][mh=0x00][len varint][digest]`, base32lower. -fn identity_locator_cid(codec: u64, digest: &[u8]) -> String { - let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); - wire.push(0x01); - write_pb_varint(codec, &mut wire); - wire.push(0x00); // multihash code: identity - write_pb_varint(digest.len() as u64, &mut wire); - wire.extend_from_slice(digest); - format!("b{}", base32_lower_no_padding(&wire)) -} - -/// Custom multicodec for the **url-locator** family: a CID whose bytes are an -/// `http(s)` URL rather than a hash of the file (METADATA_KEYS.md §2.1), so a -/// record references external bytes without carrying a raw URL field. -/// -/// Copied from the git `meta-feeder-sdk` (`hash.rs`, present since v1.1.0), -/// which this vendored copy predates. Keep the two byte-identical. -pub const URL_LOCATOR_CODEC: u64 = 0x1006; - -/// Encode an `http(s)` URL as a `0x1006` identity-multihash CIDv1. The digest -/// *is* the URL, so minting one costs no I/O. -/// -/// `None` for anything that is not an absolute http(s) URL — a relative value -/// would encode fine and then fail unresolvably at render time. -pub fn compute_url_cid(url: &str) -> Option { - let url = url.trim(); - if !(url.starts_with("http://") || url.starts_with("https://")) { - return None; - } - Some(identity_locator_cid(URL_LOCATOR_CODEC, url.as_bytes())) -} - -/// Custom multicodec for a **Usenet posting** identity (`nzb-posting`), -/// minted from the article Message-IDs of a release we scanned ourselves. -/// -/// Unlike [`NZB_RELEASE_CODEC`] (`0x1005`), this is **not** a locator: it -/// embeds no indexer host, so any peer with a plain NNTP provider can fetch the -/// articles — no `IndexerCred`, no `t=get` grab. That portability is the whole -/// point of running our own header scanner. -/// -/// It is also **not** fetchable by cid: the digest is over the Message-ID set, -/// not over any block's bytes, so bitswap can never satisfy a want for it (the -/// receiver derives a block's cid by hashing the block — see -/// `beetswap::incoming_stream`). The `.nzb` manifest travels separately, as an -/// ordinary sha2-256 IPFS cid in the record's `manifest` field. See -/// `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5. -pub const NZB_POSTING_CODEC: u64 = 0x1003; - -/// Mint an `nzb-posting` cid (`0x1003`) from a release's article Message-IDs. -/// -/// # The normalisation rule IS the dedup contract -/// -/// Mirrored verbatim in meta-share (`nzb/manifest.rs`), which re-derives this -/// digest from the fetched manifest and rejects a mismatch. **Any change here -/// must land on both sides together** — a drift silently makes every posting -/// unplayable. The rule: -/// -/// 1. Trim each id, and strip a surrounding `<…>` pair if present (NZB -/// `` bodies carry the bare form, but be defensive — NNTP `BODY` -/// re-adds the brackets). -/// 2. Drop empties. -/// 3. Deduplicate, then **sort** lexicographically by bytes. -/// 4. Join with `\n` and SHA-256 the result. -/// -/// Case is **preserved**: a Message-ID's local part is case-sensitive per RFC -/// 5322, so lowercasing could collide two genuinely distinct articles. -/// -/// Sorting is what makes the cid survive segment/file reordering between two -/// NZBs of the same posting — the reason we hash the *id set* and never the raw -/// `.nzb` bytes (the retired `0x1004` did the latter and broke dedup; see -/// meta-share `nzb/mod.rs`). -/// -/// **v1 scope:** every segment counts, par2 volumes included. Two NZBs for one -/// posting that disagree about whether to bundle par2 therefore mint different -/// cids. Acceptable while we are the only producer of these manifests; if a -/// second producer ever appears, the fix is to filter par2 subjects here *and* -/// in meta-share's mirror — do not do it on one side only. -pub fn compute_nzb_posting_cid>(message_ids: &[S]) -> String { - let mut ids: Vec<&str> = message_ids - .iter() - .map(|s| { - let t = s.as_ref().trim(); - t.strip_prefix('<') - .and_then(|r| r.strip_suffix('>')) - .unwrap_or(t) - }) - .filter(|s| !s.is_empty()) - .collect(); - ids.sort_unstable(); - ids.dedup(); - - let mut hasher = Sha256::new(); - for (i, id) in ids.iter().enumerate() { - if i > 0 { - hasher.update(b"\n"); - } - hasher.update(id.as_bytes()); - } - let digest: [u8; 32] = hasher.finalize().into(); - - // CIDv1: [version=0x01][codec varint][mh code=0x12 sha2-256][len=32][digest] - let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); - wire.push(0x01); - write_pb_varint(NZB_POSTING_CODEC, &mut wire); - wire.push(0x12); // multihash code: sha2-256 - write_pb_varint(digest.len() as u64, &mut wire); - wire.extend_from_slice(&digest); - format!("b{}", base32_lower_no_padding(&wire)) -} - -/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's -/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 -/// --hash=sha2-256 `. -pub fn compute_ipfs_cid(bytes: &[u8]) -> String { - compute_ipfs_blocks(bytes).root -} - -/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every -/// block produced along the way (leaf chunks + internal dag-pb nodes) -/// so callers can populate a bitswap blockstore in one pass. -pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks { - // Leaves: chunk bytes into raw-codec blocks. Empty input still gets - // one (empty) leaf so the cid is well-defined. - let mut blocks: Vec<(String, Bytes)> = Vec::new(); - let leaves: Vec<(Vec, u64)> = if bytes.is_empty() { - let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b"")); - blocks.push((cid_string(&cid_wire), Bytes::new())); - vec![(cid_wire, 0)] - } else { - bytes - .chunks(IPFS_CHUNK_SIZE) - .map(|chunk| { - let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk)); - blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk))); - (cid_wire, chunk.len() as u64) - }) - .collect() - }; - - // Single-leaf file: return the leaf cid as the file cid. - if leaves.len() == 1 { - let root = cid_string(&leaves[0].0); - return IpfsBlocks { root, blocks }; - } - - // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes. - let mut level: Vec<(Vec, u64, u64)> = leaves - .into_iter() - .map(|(cid, size)| (cid, size, size)) - .collect(); - - while level.len() > 1 { - let mut next: Vec<(Vec, u64, u64)> = - Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT)); - for batch in level.chunks(IPFS_FANOUT) { - let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect(); - let filesize: u64 = blocksizes.iter().sum(); - let unixfs_data = encode_unixfs_file(filesize, &blocksizes); - let node_bytes = encode_dagpb_node(batch, &unixfs_data); - let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes)); - blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone()))); - let own_tsize: u64 = - batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64; - next.push((node_cid, filesize, own_tsize)); - } - level = next; - } - let root = cid_string(&level[0].0); - IpfsBlocks { root, blocks } -} - -fn cid_string(wire: &[u8]) -> String { - format!("b{}", base32_lower_no_padding(wire)) -} - -/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code -/// varint=0x12 sha2-256][len=0x20][digest...]`. -fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec { - debug_assert!( - codec < 0x80, - "codec varint must fit in one byte for this helper" - ); - let mut wire = Vec::with_capacity(36); - wire.push(0x01); // CIDv1 - wire.push(codec); - wire.push(0x12); // multihash code: sha2-256 - wire.push(0x20); // digest length: 32 - wire.extend_from_slice(digest); - wire -} - -fn sha2_256(bytes: &[u8]) -> [u8; 32] { - Sha256::digest(bytes).into() -} - -/// LEB128 unsigned varint encoding into `out`. -fn write_pb_varint(value: u64, out: &mut Vec) { - let mut v = value; - while v >= 0x80 { - out.push((v as u8) | 0x80); - v >>= 7; - } - out.push(v as u8); -} - -fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) { - write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint) - write_pb_varint(value, out); -} - -fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) { - write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited - write_pb_varint(value.len() as u64, out); - out.extend_from_slice(value); -} - -/// UnixFS protobuf payload for a File node: `Type=File, filesize, -/// blocksizes[]`. -fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec { - let mut out = Vec::new(); - write_pb_varint_field(1, 2, &mut out); // Type = File - write_pb_varint_field(3, filesize, &mut out); // filesize - for &bs in blocksizes { - write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated) - } - out -} - -/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order -/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second. -fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec { - let mut out = Vec::new(); - for (cid_wire, _filesize, tsize) in children { - let mut link_bytes = Vec::new(); - write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash - write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize - write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first - } - write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last - out -} - -/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the -/// multibase `b` prefix. -pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { - const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; - let mut out = String::with_capacity(input.len().div_ceil(5) * 8); - let mut buffer: u64 = 0; - let mut bits: u32 = 0; - for &b in input { - buffer = (buffer << 8) | b as u64; - bits += 8; - while bits >= 5 { - bits -= 5; - let idx = ((buffer >> bits) & 0x1F) as usize; - out.push(ALPHABET[idx] as char); - } - } - if bits > 0 { - let idx = ((buffer << (5 - bits)) & 0x1F) as usize; - out.push(ALPHABET[idx] as char); - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Pinned against an independent encoder. The tmdb plugin's TS - /// `encodeUrlCid` asserts the same strings, so a drift on either side fails. - /// The long URL exercises the two-byte length varint (> 127 bytes). - #[test] - fn url_cid_matches_pinned_vectors() { - assert_eq!( - compute_url_cid("https://image.tmdb.org/t/p/w500/dqZENchTd7lp5zit1Q7Bkjzcxpi.jpg") - .as_deref(), - Some("bagdcaab7nb2hi4dthixs62lnmftwkltunvsgeltpojts65bpoaxxonjqgaxwi4k2ivhgg2cumq3wy4bvpjuximkrg5bgw2t2mn4ha2jonjygo") - ); - let long = format!("https://image.tmdb.org/t/p/w500/{}.jpg", "a".repeat(100)); - assert_eq!(long.len(), 136); - assert_eq!( - compute_url_cid(&long).as_deref(), - Some("bagdcaaeiafuhi5dqom5c6l3jnvqwozjoorwwiyron5zgol3uf5yc65zvgayc6ylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcylbmfqwcltkobtq") - ); - assert!(compute_url_cid("/relative.jpg").is_none()); - assert!(compute_url_cid("ftp://example.com/x.jpg").is_none()); - } - - #[test] - fn midhash256_matches_ts_fixtures() { - assert_eq!( - compute_midhash256(b"hello world"), - "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4" - ); - assert_eq!( - compute_midhash256(b""), - "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y" - ); - let mb_zeros = vec![0u8; 1024 * 1024]; - assert_eq!( - compute_midhash256(&mb_zeros), - "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4" - ); - } - - #[test] - fn midhash256_from_sample_matches_full_bytes() { - let small: &[u8] = b"hello world"; - assert_eq!( - compute_midhash256_from_sample(small.len() as u64, small), - compute_midhash256(small), - ); - - assert_eq!( - compute_midhash256_from_sample(0, b""), - compute_midhash256(b""), - ); - - let one_mb = vec![0u8; SAMPLE_SIZE]; - assert_eq!( - compute_midhash256_from_sample(one_mb.len() as u64, &one_mb), - compute_midhash256(&one_mb), - ); - - let two_mb = vec![0u8; 2 * SAMPLE_SIZE]; - let start = (two_mb.len() - SAMPLE_SIZE) / 2; - let middle = &two_mb[start..start + SAMPLE_SIZE]; - assert_eq!( - compute_midhash256_from_sample(two_mb.len() as u64, middle), - compute_midhash256(&two_mb), - ); - } - - #[test] - fn midhash256_from_sample_partial_sample_stable() { - let sample = vec![0u8; SAMPLE_SIZE]; - let total_5gb: u64 = 5 * 1024 * 1024 * 1024; - let total_10gb: u64 = 10 * 1024 * 1024 * 1024; - - let h1 = compute_midhash256_from_sample(total_5gb, &sample); - let h2 = compute_midhash256_from_sample(total_5gb, &sample); - assert_eq!(h1, h2, "same (size, sample) must hash identically"); - - let h3 = compute_midhash256_from_sample(total_10gb, &sample); - assert_ne!( - h1, h3, - "size prefix must contribute to cid — same sample, different size" - ); - } - - #[test] - fn bt_v1_file_cid_wire_layout() { - let infohash: [u8; 20] = [ - 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c, - 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06, - ]; - - let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; - want.extend_from_slice(&infohash); - want.push(0x04); - assert_eq!( - compute_bt_v1_file_cid(&infohash, 4), - format!("b{}", base32_lower_no_padding(&want)) - ); - - let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; - want0.extend_from_slice(&infohash); - want0.push(0x00); - assert_eq!( - compute_bt_v1_file_cid(&infohash, 0), - format!("b{}", base32_lower_no_padding(&want0)) - ); - - let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16]; - want200.extend_from_slice(&infohash); - want200.extend_from_slice(&[0xC8, 0x01]); - assert_eq!( - compute_bt_v1_file_cid(&infohash, 200), - format!("b{}", base32_lower_no_padding(&want200)) - ); - } - - #[test] - fn bt_v2_file_cid_wire_layout() { - let infohash: [u8; 32] = [0xAB; 32]; - let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21]; - want.extend_from_slice(&infohash); - want.push(0x00); - assert_eq!( - compute_bt_v2_file_cid(&infohash, 0), - format!("b{}", base32_lower_no_padding(&want)) - ); - } - - #[test] - fn bt_file_cid_properties() { - let ih1 = [0x11u8; 20]; - let ih2 = [0x22u8; 32]; - - assert_eq!( - compute_bt_v1_file_cid(&ih1, 3), - compute_bt_v1_file_cid(&ih1, 3) - ); - assert_ne!( - compute_bt_v1_file_cid(&ih1, 0), - compute_bt_v1_file_cid(&ih1, 1) - ); - assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b')); - assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b')); - assert_ne!( - compute_bt_v1_file_cid(&[0xAB; 20], 0), - compute_bt_v2_file_cid(&[0xAB; 32], 0) - ); - } - - #[test] - fn base32_known_vectors() { - assert_eq!(base32_lower_no_padding(b""), ""); - assert_eq!(base32_lower_no_padding(b"f"), "my"); - assert_eq!(base32_lower_no_padding(b"fo"), "mzxq"); - assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6"); - assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq"); - assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi"); - } - - #[test] - fn yt_video_cid_matches_golden_vector() { - // Pinned by `/cid-rank-vectors.json`'s `yt-video-locator` entry. The id - // is the real resolver output for Daft Punk / "Get Lucky" measured in - // the delegated-playback study §4.3. - assert_eq!( - compute_yt_video_cid("video", "4D7u5KF7SP8").unwrap(), - "bagecaaarav3gszdfn42ein3vgvfumn2tka4a" - ); - } - - #[test] - fn yt_video_cid_framing_matches_card_locator() { - // The two codecs share the `varint(len(kind)) || kind || id` framing on - // purpose, so every decoder extends a parser it already wrote. If this - // ever fails, the two shapes have drifted and §7.2 of docs/cid-formats.md - // is no longer true. - let yt = compute_yt_video_cid("tmdb", "tv:95479").unwrap(); - let card = compute_card_cid("tmdb", "tv:95479").unwrap(); - // Same digest, different codec => same length, differing only in the - // codec varint region. - assert_ne!(yt, card); - assert_eq!(yt.len(), card.len()); - } - - #[test] - fn yt_video_cid_is_a_pure_function_of_kind_and_id() { - // Convergence is the reason this is a codec and not a URL: two peers - // that resolve the same video must derive the same address offline. - assert_eq!( - compute_yt_video_cid("video", "4D7u5KF7SP8"), - compute_yt_video_cid("video", "4D7u5KF7SP8") - ); - // ...and the kind is part of the identity, not decoration. - assert_ne!( - compute_yt_video_cid("video", "OLAK5uy_kZ8Xq"), - compute_yt_video_cid("playlist", "OLAK5uy_kZ8Xq") - ); - } - - #[test] - fn yt_video_cid_rejects_empty_fields() { - assert!(compute_yt_video_cid("", "4D7u5KF7SP8").is_none()); - assert!(compute_yt_video_cid("video", "").is_none()); - } - - #[test] - fn ext_play_cid_matches_golden_vector() { - assert_eq!( - compute_ext_play_cid("https://example.bandcamp.com/album/x").unwrap(), - "bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa" - ); - } - - #[test] - fn ext_play_cid_rejects_non_http_schemes() { - // Load-bearing: a client hands this string to an anchor or a new tab, - // so a `javascript:` or `data:` payload must never mint a locator. - assert!(compute_ext_play_cid("javascript:alert(1)").is_none()); - assert!(compute_ext_play_cid("data:text/html,