From d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 24 Aug 2026 16:31:56 +0200 Subject: [PATCH] rework plugin --- .gitattributes | 6 + .gitignore | 3 + Cargo.toml | 63 ++ README.md | 139 ++++- rust-toolchain.toml | 3 + src/cache.rs | 353 ++++++++++++ src/common.rs | 288 ++++++++++ src/config.rs | 312 ++++++++++ src/config_page.html | 268 +++++++++ src/enrich.rs | 614 ++++++++++++++++++++ src/filename_meta.rs | 1309 ++++++++++++++++++++++++++++++++++++++++++ src/hash.rs | 1076 ++++++++++++++++++++++++++++++++++ src/lang.rs | 81 +++ src/lib.rs | 49 ++ src/meta_core.rs | 335 +++++++++++ src/plugin.rs | 258 +++++++++ src/query.rs | 147 +++++ src/query_eval.rs | 514 +++++++++++++++++ src/serve.rs | 762 ++++++++++++++++++++++++ src/types.rs | 124 ++++ 20 files changed, 6703 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 rust-toolchain.toml create mode 100644 src/cache.rs create mode 100644 src/common.rs create mode 100644 src/config.rs create mode 100644 src/config_page.html create mode 100644 src/enrich.rs create mode 100644 src/filename_meta.rs create mode 100644 src/hash.rs create mode 100644 src/lang.rs create mode 100644 src/lib.rs create mode 100644 src/meta_core.rs create mode 100644 src/plugin.rs create mode 100644 src/query.rs create mode 100644 src/query_eval.rs create mode 100644 src/serve.rs create mode 100644 src/types.rs diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..022a2bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +Cargo.lock +.cargo/config.toml diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..660ba1e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "meta-feeder-sdk" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-sdk" +description = "Source-agnostic foundation for MetaMesh gateway feeders: the FeederPlugin trait, the content-addressing hash families, the structured-query evaluator, and an axum harness that serves any set of plugins over the feeder HTTP contract." +readme = "README.md" +keywords = ["metamesh", "feeder", "gateway", "content-addressing", "plugin"] +categories = ["api-bindings", "web-programming::http-server"] + +# Not on crates.io yet — consumers depend on a git tag while the contract is +# still moving (see docs/project-architecture/feeder-architecture.md §8). +# Flip this, not the dep form, when `cargo add meta-feeder-sdk` should work. +publish = false + +# This repo IS the crate. The explicit (empty) workspace table stops it being +# absorbed into a parent workspace when it is vendored or path-overridden into +# a feeder checkout during local development. +[workspace] + +# Source-agnostic foundation shared by the gateway core and every feeder +# sidecar binary: the `FeederPlugin` trait, the content-addressing hash +# families, the per-plugin redb cache, the shared HTTP/cache helpers, the +# structured-query evaluator, and the axum `serve_feeders` harness that wraps +# any set of `FeederPlugin`s in the feeder HTTP contract. +# +# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes; +# the gateway core owns hashing-into-the-blockstore, the meta-core store-back, +# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT +# of here so feeders that don't need them don't pay for them. + +[dependencies] +# Async runtime + HTTP server (the serve_feeders harness) +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +hyper = { version = "1", features = ["full"] } +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" +base64 = "0.22" + +# Utilities +futures = "0.3" +async-trait = "0.1" +anyhow = "1" +thiserror = "1" +tracing = "0.1" +sha2 = "0.10" +bytes = "1" +# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). +regex = "1" + +# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see +# cache.rs. Pure-Rust single-process embedded store. +redb = "2" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/README.md b/README.md index 853f5b3..9756eb6 100644 --- a/README.md +++ b/README.md @@ -1 +1,138 @@ -# meta-feeder-sdk \ No newline at end of file +# meta-feeder-sdk + +The source-agnostic foundation shared by the MetaMesh gateway core and every +feeder sidecar. + +A **feeder** bridges one external source into the MetaMesh network. It +implements [`FeederPlugin`](src/plugin.rs) — find records, resolve them to +content-addressed outcomes, optionally fetch bytes — and `serve_feeders` exposes +any set of plugins over the feeder HTTP contract. The gateway core consumes that +contract and keeps the libp2p wire, the bitswap blockstore, the +hashing-into-the-blockstore and the meta-core store-back to itself. + +Deliberately **libp2p-free and blockstore-free**: a feeder finds and fetches +bytes; it never joins the swarm. + +## Using it + +```toml +[dependencies] +meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v0.1.0" } +``` + +Not on crates.io yet — the contract is still moving, so consumers pin a tag. + +```rust +use meta_feeder_sdk::{serve_feeders, FeederPlugin}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let plugins: Vec> = vec![Box::new(MyPlugin::new())]; + serve_feeders(plugins, "/data/meta-feeder".into(), "0.0.0.0:8080".parse()?).await +} +``` + +A **service is a `Vec`** — grouping plugins into one binary is a deployment +choice, not an architectural one. + +## Writing a plugin + +Four required methods; everything else has a default: + +| method | you must | notes | +|---|---|---| +| `upstream_id` | return a stable `&'static str` | identifies your records | +| `configure` | set up a cache dir; return `MissingConfig` if unconfigured | the harness soft-skips a plugin that returns `MissingConfig`, so an unconfigured plugin degrades instead of killing the sidecar | +| `handle_query` | answer a `GatewayQuery` with `DiscoveryRecord`s | **open with `query_eval::query_accepts_plugin`** — see below | +| `compute_outcomes` | resolve a `record_id` to `HashOutcome`s | `bytes: None` for metadata-only sources | + +Defaults you can override: `handle_query_stream`, `handle_fetch`, `health`, +`served_file_types`, `served_content_kinds`, `config_schema`, `config_values`, +`get_blob`. + +### The routing gate + +The gateway dispatches on `(fileType, contentKind)`, never on a service name. +Advertise your axes and gate on them, or you will be handed queries you cannot +answer: + +```rust +fn served_file_types(&self) -> &'static [&'static str] { &["video"] } +fn served_content_kinds(&self) -> &'static [&'static str] { &["movie", "series"] } + +async fn handle_query(&self, q: &GatewayQuery, n: usize) -> Result, GatewayError> { + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + q, self.served_file_types(), self.served_content_kinds()) { + return Ok(Vec::new()); + } + // ... +} +``` + +`&["*"]` on an axis means "I serve every value on it". + +### Three rules that are not obvious + +1. **Records must carry a type filter.** A record with no `fileType` / + `contentKind` never reaches a client wall, and the symptom looks like a + transport bug rather than a metadata one. +2. **`filters` / `ranges` / `negations` are authoritative; `raw_text` is + informational.** Consume one source per filter or you will double-filter and + under-match. +3. **Never degrade an unresolvable anchor into a free-text query.** + `free_text_or_star()` exists for upstreams that 4xx on empty input — reaching + for it when an *identity* filter failed to resolve dumps the upstream's + recent feed into someone's library. + +## Hash families + +`HashKind` selects how the core routes an outcome. Only `Sha2_256` is seeded +into the bitswap blockstore. + +| kind | codec | means | +|---|---|---| +| `Sha2_256` | standard | real bytes, retrievable via bitswap | +| `Midhash256` | `0x1000` | size + middle-sample; MetaMesh-internal | +| `BtV1File` | `0x1001` | a file inside a torrent — locator | +| `NzbRelease` | `0x1005` | a Usenet release — locator, redeemed by a credentialed peer | +| `CardLocator` | `0x1007` | a *work*, no bytes anywhere | +| `YtVideo` | `0x1008` | **delegated playback** — bytes are permanently someone else's | + +The last two carry an identity multihash. Anything that dispatches on the +multihash instead of the **codec** will silently mis-rank them. + +## Layout + +| module | what | +|---|---| +| `plugin` | the `FeederPlugin` trait, `HashKind`, `HashOutcome`, `ConfigError` | +| `types` | `DiscoveryRecord`, `GatewayError`, `PluginHealth`, `ByteStream` | +| `query` / `query_eval` | the structured query and its evaluator | +| `serve` | the axum harness and the HTTP contract DTOs | +| `config` | self-describing config schema — the dashboard renders a form from it | +| `cache` | per-plugin redb cache | +| `hash` | the content-addressing families above | +| `filename_meta` / `lang` | release-tag parsing helpers | +| `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract | + +## Development + +```bash +cargo build && cargo test +``` + +To work on the SDK from inside a feeder checkout without pushing a tag, add a +gitignored `.cargo/config.toml` in the feeder: + +```toml +paths = ["../meta-feeder-sdk"] +``` + +## Context + +Design rationale, the feeder roster, and the naming rules live in the meta-root: +`docs/project-architecture/feeder-architecture.md`. + +## License + +MIT diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"] diff --git a/src/cache.rs b/src/cache.rs new file mode 100644 index 0000000..e04aae4 --- /dev/null +++ b/src/cache.rs @@ -0,0 +1,353 @@ +//! Per-plugin midhash cache backed by redb. +//! +//! Each plugin gets its own redb file at +//! `/gateway//cache.redb`. Keys are upstream +//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the +//! `Midhash` strings the plugin computed by fetching + hashing the upstream +//! file. +//! +//! The schema is identical to meta-share v1's so old `.redb` files copy +//! across. redb is synchronous and sub-millisecond for small ops, so calls +//! run inline from async handlers without `spawn_blocking`. + +use std::path::Path; +use std::sync::Arc; + +use redb::{Database, ReadableTableMetadata, TableDefinition}; + +pub const CACHE_FILENAME: &str = "cache.redb"; + +const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash"); +const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs"); +const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid"); +const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec"); +const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid"); +/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload +/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the +/// hash families differ: `midhash` is the synthetic midhash256-from-infohash +/// fallback; `fullhash` records "we actually downloaded the bytes and this is +/// the IPFS CIDv1 over them." +const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash"); +/// Enumerated torrent file list (JSON `Vec`) keyed by the +/// torrent's `record_id` (infohash). +const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist"); +/// Cached TMDB *search* results keyed by a stable lookup key. The value is the +/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache +/// sentinel. +const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search"); +/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`. +const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_tvdetails"); +/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`. +const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids"); +/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`. +const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_moviedetails"); +/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized +/// free-text query. +const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_principal_topn"); +/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's +/// `record_id` (infohash). Value is the JSON encoding of a `Vec`. +const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles"); +/// OpenSubtitles search results keyed by `"\x01"`. Value is the +/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel. +const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles"); +/// Small durable plugin scratch values that must survive a restart but aren't +/// content-derived — today, the torznab plugin's Prowlarr indexer-set +/// fingerprint, so a set that changed *while the feeder was down* is still +/// detected as a change on the next boot. Keys are plugin-namespaced +/// (`prowlarr:indexer-set`). redb creates the table on open, so an older +/// `.redb` file gains it transparently. +const MISC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("misc"); + +/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared +/// via `Arc`). +#[derive(Clone)] +pub struct MidhashCache { + db: Arc, +} + +/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb +/// table. +macro_rules! str_table_accessors { + ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => { + $(#[$gmeta])* + pub fn $get(&self, key: &str) -> Result, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table($table)?; + Ok(table.get(key)?.map(|v| v.value().to_string())) + } + + $(#[$pmeta])* + pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table($table)?; + table.insert(key, value)?; + } + tx.commit()?; + Ok(()) + } + }; +} + +impl MidhashCache { + pub fn open(cache_dir: &Path) -> Result { + let path = cache_dir.join(CACHE_FILENAME); + let db = Database::create(&path)?; + { + let tx = db.begin_write()?; + tx.open_table(MIDHASH_TABLE)?; + tx.open_table(BLOBS_TABLE)?; + tx.open_table(COVER_CID_TABLE)?; + tx.open_table(BIBREC_TABLE)?; + tx.open_table(PREVIEW_CID_TABLE)?; + tx.open_table(FULLHASH_TABLE)?; + tx.open_table(FILELIST_TABLE)?; + tx.open_table(TMDB_SEARCH_TABLE)?; + tx.open_table(TMDB_TVDETAILS_TABLE)?; + tx.open_table(TMDB_EXTIDS_TABLE)?; + tx.open_table(TMDB_MOVIEDETAILS_TABLE)?; + tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?; + tx.open_table(SUBTITLES_TABLE)?; + tx.open_table(OPENSUBTITLES_TABLE)?; + tx.open_table(MISC_TABLE)?; + tx.commit()?; + } + Ok(Self { db: Arc::new(db) }) + } + + str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE); + + str_table_accessors!( + /// Read a durable plugin scratch value (see [`MISC_TABLE`]). + get_misc, + /// Write a durable plugin scratch value (see [`MISC_TABLE`]). + put_misc, + MISC_TABLE, + ); + + pub fn entry_count(&self) -> Result { + let tx = self.db.begin_read()?; + let table = tx.open_table(MIDHASH_TABLE)?; + Ok(table.len()?) + } + + pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BLOBS_TABLE)?; + Ok(table.get(cid)?.map(|v| v.value().to_vec())) + } + + pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BLOBS_TABLE)?; + table.insert(cid, bytes)?; + } + tx.commit()?; + Ok(()) + } + + str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE); + + str_table_accessors!( + /// Read a previously-stored full-file IPFS cid for `record_id`. + get_fullhash, + /// Record the IPFS cid produced by a successful full fetch. + put_fullhash, + FULLHASH_TABLE + ); + + str_table_accessors!( + /// Read the cached torrent file list (JSON) for `record_id`. + get_filelist, + /// Cache the enumerated torrent file list (JSON `Vec`). + put_filelist, + FILELIST_TABLE + ); + + str_table_accessors!( + /// Read the cached subtitle linkage (JSON `Vec`). + get_subtitles, + /// Cache the discovered subtitle linkage (JSON `Vec`). + put_subtitles, + SUBTITLES_TABLE + ); + + str_table_accessors!( + /// Read a cached OpenSubtitles lookup by `"\x01"`. + get_opensubtitles, + /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel). + put_opensubtitles, + OPENSUBTITLES_TABLE + ); + + str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE); + + str_table_accessors!( + /// Read a cached TMDB search result by lookup key. + get_tmdb_search, + /// Cache a TMDB search result under `key`. + put_tmdb_search, + TMDB_SEARCH_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB TV-details JSON by `tmdbid`. + get_tmdb_tvdetails, + /// Cache TMDB TV-details JSON under `tmdbid`. + put_tmdb_tvdetails, + TMDB_TVDETAILS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB external-ids JSON by `tmdbid`. + get_tmdb_extids, + /// Cache TMDB external-ids JSON under `tmdbid`. + put_tmdb_extids, + TMDB_EXTIDS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB movie-details JSON by `tmdbid`. + get_tmdb_moviedetails, + /// Cache TMDB movie-details JSON under `tmdbid`. + put_tmdb_moviedetails, + TMDB_MOVIEDETAILS_TABLE + ); + + str_table_accessors!( + /// Read the cached ranked anchor list by normalized query key. + get_tmdb_principal_topn, + /// Cache the ranked anchor list under the normalized query key. + put_tmdb_principal_topn, + TMDB_PRINCIPAL_TOPN_TABLE + ); + + pub fn get_bibrec( + &self, + record_id: &str, + ) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BIBREC_TABLE)?; + let Some(v) = table.get(record_id)? else { + return Ok(None); + }; + let s = v.value(); + Ok(serde_json::from_str(s).ok()) + } + + pub fn put_bibrec( + &self, + record_id: &str, + fields: &std::collections::BTreeMap, + ) -> Result<(), redb::Error> { + let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()); + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BIBREC_TABLE)?; + table.insert(record_id, json.as_str())?; + } + tx.commit()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_cache_dir() -> tempfile::TempDir { + tempfile::tempdir().expect("tempdir") + } + + #[test] + fn put_then_get_returns_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None); + cache.put_midhash("10.1038/x", "bafyXYZ").expect("put"); + assert_eq!( + cache.get_midhash("10.1038/x").unwrap().as_deref(), + Some("bafyXYZ") + ); + } + + #[test] + fn put_overwrites_existing_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + cache.put_midhash("k", "v1").unwrap(); + cache.put_midhash("k", "v2").unwrap(); + assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2")); + } + + #[test] + fn values_persist_across_reopens() { + let dir = tmp_cache_dir(); + { + let cache = MidhashCache::open(dir.path()).expect("open 1"); + cache.put_midhash("k", "persistent").unwrap(); + } + let cache = MidhashCache::open(dir.path()).expect("open 2"); + assert_eq!( + cache.get_midhash("k").unwrap().as_deref(), + Some("persistent") + ); + } + + #[test] + fn tmdb_search_cache_distinguishes_miss_from_absent() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None); + cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap(); + assert_eq!( + cache + .get_tmdb_search("m\u{1}naruto\u{1}") + .unwrap() + .as_deref(), + Some("null") + ); + cache + .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}") + .unwrap(); + drop(cache); + let cache = MidhashCache::open(dir.path()).expect("reopen"); + assert_eq!( + cache + .get_tmdb_search("t\u{1}spy x family\u{1}2019") + .unwrap() + .as_deref(), + Some("{\"tmdbid\":120089}") + ); + } + + #[test] + fn tmdb_tvdetails_cache_roundtrips() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None); + cache + .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}") + .unwrap(); + assert_eq!( + cache.get_tmdb_tvdetails("120089").unwrap().as_deref(), + Some("{\"number_of_seasons\":2}") + ); + } + + #[test] + fn entry_count_reflects_inserts() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.entry_count().unwrap(), 0); + cache.put_midhash("a", "x").unwrap(); + cache.put_midhash("b", "y").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + cache.put_midhash("a", "x2").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + } +} diff --git a/src/common.rs b/src/common.rs new file mode 100644 index 0000000..c7a09f5 --- /dev/null +++ b/src/common.rs @@ -0,0 +1,288 @@ +//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`, +//! `pubmed`, `wikicommons`, `giphy`, `scihub`). +//! +//! These plugins are structurally near-identical: build a reqwest client, map +//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose +//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding +//! lives here; the genuinely-different bodies stay in the plugin modules. + +use std::path::Path; +use std::time::Duration; + +use tracing::{debug, warn}; + +use crate::cache::MidhashCache; +use crate::plugin::{ConfigError, HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, GatewayError, Hash}; + +/// Build the shared reqwest client every source plugin uses. +pub fn build_http_client( + timeout_secs: u64, + user_agent: &str, + redirect: Option, +) -> reqwest::Client { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(timeout_secs)) + .user_agent(user_agent); + if let Some(policy) = redirect { + builder = builder.redirect(policy); + } + builder + .build() + .expect("reqwest client build is infallible with rustls-tls") +} + +/// Map a non-success HTTP status to a [`GatewayError`]. +/// +/// - `404` → [`GatewayError::NotFound`] +/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s) +/// - `5xx` → [`GatewayError::Transient`] +/// - anything else → [`GatewayError::Permanent`] +pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> { + let status = resp.status(); + if status.is_success() { + return Ok(()); + } + let code = status.as_u16(); + if code == 404 { + return Err(GatewayError::NotFound); + } + if code == 429 { + let retry_after = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(60); + return Err(GatewayError::RateLimited { + retry_after_s: retry_after, + }); + } + if status.is_server_error() { + return Err(GatewayError::Transient(format!( + "upstream {} returned {status}", + resp.url() + ))); + } + Err(GatewayError::Permanent(format!( + "upstream {} returned {status}", + resp.url() + ))) +} + +/// Open the per-plugin redb midhash cache, wrapping the error with the plugin +/// name for a clean startup diagnostic. +pub fn open_midhash_cache( + cache_dir: &Path, + plugin: &'static str, +) -> Result { + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin, + source: anyhow::anyhow!("open redb cache: {e}"), + }) +} + +/// Cache-hit fast path for single-outcome plugins. +/// +/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record — +/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a +/// cache miss; the caller proceeds to fetch + hash. +pub fn cached_outcome( + cache: &MidhashCache, + record_id: &str, + upstream_id: &'static str, +) -> Result>, GatewayError> { + let Some(cached) = cache + .get_midhash(record_id) + .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))? + else { + return Ok(None); + }; + debug!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + "hash cache hit" + ); + Ok(Some(vec![HashOutcome { + hash: Hash(cached), + hash_kind: HashKind::Sha2_256, + bytes: None, + record: None, + file_extension: None, + }])) +} + +/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is +/// non-fatal, so it is logged and swallowed. +pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) { + if let Err(e) = cache.put_midhash(record_id, cid) { + warn!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + error = %e, + "hash cache put failed (non-fatal)" + ); + } +} + +/// Resolve the per-plugin cache handle, or produce the standard "not +/// configured" internal error. +pub fn require_cache<'a>( + cache: Option<&'a MidhashCache>, + plugin: &str, +) -> Result<&'a MidhashCache, GatewayError> { + cache.ok_or_else(|| { + GatewayError::Internal(anyhow::anyhow!( + "{plugin} plugin not configured (configure() never called)" + )) + }) +} + +/// Build the single-element outcome vec a single-outcome plugin returns on a +/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID, +/// and an optional file extension. +pub fn single_outcome( + cid: String, + bytes: bytes::Bytes, + record: DiscoveryRecord, + file_extension: Option, +) -> Vec { + vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record: Some(record), + file_extension, + }] +} + +/// Percent-encode a query-string value using the unreserved set +/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space. +pub fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.as_bytes() { + match *b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*b as char); + } + b' ' => out.push('+'), + other => out.push_str(&format!("%{:02X}", other)), + } + } + out +} + +/// GET `url` and return the body bytes, mapping transport + status errors to +/// [`GatewayError`]. +pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.bytes() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Like [`fetch_bytes`] but decodes the body as UTF-8 text. +pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.text() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Decode a v1 BitTorrent infohash string into its raw 20 bytes. +/// +/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648, +/// case-insensitive). Returns `None` for anything else. +pub fn decode_infohash(s: &str) -> Option<[u8; 20]> { + let bytes = s.as_bytes(); + let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) { + let mut out = Vec::with_capacity(20); + for pair in bytes.chunks(2) { + out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?); + } + out + } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) { + decode_base32_20(bytes)? + } else { + return None; + }; + raw.try_into().ok() +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20 +/// bytes they encode. +fn decode_base32_20(chars: &[u8]) -> Option> { + let mut out = Vec::with_capacity(20); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &c in chars { + let val = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a', + b'2'..=b'7' => c - b'2' + 26, + _ => return None, + } as u64; + buffer = (buffer << 5) | val; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push((buffer >> bits) as u8); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_infohash_hex_roundtrips() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let raw = decode_infohash(hex).expect("hex decodes"); + assert_eq!(raw[0], 0x01); + assert_eq!(raw[19], 0x67); + } + + #[test] + fn decode_infohash_is_case_insensitive_hex() { + let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd"); + let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD"); + assert!(lower.is_some()); + assert_eq!(lower, upper); + } + + #[test] + fn decode_infohash_accepts_base32() { + let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U"); + assert!(raw.is_some()); + } + + #[test] + fn decode_infohash_rejects_wrong_shape() { + assert!(decode_infohash("tooshort").is_none()); + assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none()); + assert!(decode_infohash("").is_none()); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..c4af5d6 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,312 @@ +//! Self-describing per-plugin configuration: schema, redaction, merge, and the +//! generic schema-driven HTML page the feeder serves at `/config`. +//! +//! The point of this module is that **the gateway carries no per-plugin config +//! knowledge** and **the SDK carries no per-plugin field names**. A plugin +//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema); +//! everything here (redaction of secrets, merge-on-save, and the rendered form) +//! is driven entirely by that schema. Add a feeder → it shows up with a working +//! config form and zero UI/gateway changes. +//! +//! ## Persistence & precedence +//! +//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to +//! `configure()`). The plugin reads **file-or-env** at startup: env is the +//! first-boot seed and stays authoritative until the operator saves through the +//! UI, at which point `config.json` exists and wins. There is no runtime reload +//! — a save returns `restart_required: true` (mirrors the gateway's +//! `gateway-config.json` posture). +//! +//! ## Secrets +//! +//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to +//! the page (redacted to a `_set: bool`), and a blank incoming value on +//! save means "keep the stored secret". + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A plugin's full config surface — an ordered list of fields. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ConfigSchema { + pub fields: Vec, +} + +impl ConfigSchema { + /// Convenience: `true` when the plugin declares no configurable fields (the + /// page then renders a "no configuration required" notice). + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } +} + +/// One configurable field. `record_array` fields nest `fields` for the +/// per-record sub-schema (e.g. torznab's `indexers`). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ConfigField { + /// JSON key in the config object. + pub key: String, + /// Human label for the form. + pub label: String, + #[serde(rename = "type")] + pub kind: FieldKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub help: Option, + #[serde(default)] + pub required: bool, + /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated + /// as the record identifier for secret back-fill on merge (e.g. `name`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +impl ConfigField { + pub fn text(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Text) + } + pub fn secret(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Secret) + } + pub fn bool(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Bool) + } + pub fn number(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Number) + } + pub fn list(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::List) + } + pub fn record_array(key: &str, label: &str, fields: Vec) -> Self { + let mut f = Self::new(key, label, FieldKind::RecordArray); + f.fields = Some(fields); + f + } + fn new(key: &str, label: &str, kind: FieldKind) -> Self { + Self { + key: key.to_string(), + label: label.to_string(), + kind, + help: None, + required: false, + fields: None, + } + } + pub fn with_help(mut self, help: &str) -> Self { + self.help = Some(help.to_string()); + self + } + pub fn required(mut self) -> Self { + self.required = true; + self + } +} + +/// Field widget / value type. Drives both rendering and the secret/merge rules. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FieldKind { + /// Free text → JSON string. + Text, + /// Write-only secret → JSON string; redacted on read, "blank keeps" on save. + Secret, + /// Checkbox → JSON bool. + Bool, + /// Numeric input → JSON number. + Number, + /// Comma/line list → JSON array of strings. + List, + /// Repeatable record → JSON array of objects keyed by nested `fields`. + RecordArray, +} + +fn is_blank(v: Option<&Value>) -> bool { + match v { + None | Some(Value::Null) => true, + Some(Value::String(s)) => s.is_empty(), + _ => false, + } +} + +/// Produce a UI-safe view of `values`: secret fields are dropped and replaced +/// with a `_set` boolean; everything else passes through. Recurses into +/// `record_array` records for secret sub-fields. +pub fn redact(values: &Value, schema: &ConfigSchema) -> Value { + let src = values.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&src, &schema.fields)) +} + +fn redact_object(src: &Map, fields: &[ConfigField]) -> Map { + let mut out = Map::new(); + for f in fields { + match f.kind { + FieldKind::Secret => { + let set = !is_blank(src.get(&f.key)); + out.insert(format!("{}_set", f.key), Value::Bool(set)); + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let arr = src + .get(&f.key) + .and_then(Value::as_array) + .map(|recs| { + recs.iter() + .map(|r| { + let m = r.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&m, subs)) + }) + .collect::>() + }) + .unwrap_or_default(); + out.insert(f.key.clone(), Value::Array(arr)); + } + _ => { + if let Some(v) = src.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// Merge `incoming` (from the UI) onto `base` (the stored/effective config), +/// schema-aware. Returns the full, UNREDACTED object to persist: +/// - secret: blank/absent incoming keeps `base`; non-blank overwrites. +/// - text/number/bool/list: present in incoming overwrites; absent keeps base. +/// - record_array: incoming array replaces base, but blank secret sub-fields are +/// back-filled from the base record matched by the first sub-field (identifier). +pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value { + let base_obj = base.as_object().cloned().unwrap_or_default(); + let inc_obj = incoming.as_object().cloned().unwrap_or_default(); + Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields)) +} + +fn merge_object( + base: &Map, + inc: &Map, + fields: &[ConfigField], +) -> Map { + let mut out = base.clone(); + for f in fields { + match f.kind { + FieldKind::Secret => { + if !is_blank(inc.get(&f.key)) { + out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap()); + } + // blank/absent → keep base (no-op) + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let id_key = subs.first().map(|s| s.key.clone()); + let base_recs = base.get(&f.key).and_then(Value::as_array).cloned(); + let inc_recs = match inc.get(&f.key).and_then(Value::as_array) { + Some(r) => r.clone(), + None => continue, // absent → keep base array untouched + }; + let merged: Vec = inc_recs + .iter() + .map(|rec| { + let rec_obj = rec.as_object().cloned().unwrap_or_default(); + // Find the matching base record by identifier sub-field. + let base_match = match (&id_key, base_recs.as_ref()) { + (Some(idk), Some(brs)) => brs + .iter() + .find(|br| br.get(idk) == rec_obj.get(idk)) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(), + _ => Map::new(), + }; + Value::Object(merge_object(&base_match, &rec_obj, subs)) + }) + .collect(); + out.insert(f.key.clone(), Value::Array(merged)); + } + _ => { + if let Some(v) = inc.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// The generic, schema-driven config page served at `GET /config`. Self-contained +/// (inline CSS + JS, no external assets) so it loads identically when hit +/// directly on the feeder OR reverse-proxied through the gateway. All API calls +/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant. +pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html"); + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn torznab_like() -> ConfigSchema { + ConfigSchema { + fields: vec![ + ConfigField::secret("tmdb_token", "TMDB token"), + ConfigField::bool("resolve_torrent_files", "Resolve .torrent"), + ConfigField::record_array( + "indexers", + "Indexers", + vec![ + ConfigField::text("name", "Name"), + ConfigField::secret("api_key", "API key"), + ], + ), + ], + } + } + + #[test] + fn redact_hides_secrets_and_flags_set() { + let v = json!({ + "tmdb_token": "abc", + "resolve_torrent_files": true, + "indexers": [{"name":"prowlarr","api_key":"k"}] + }); + let r = redact(&v, &torznab_like()); + assert_eq!(r["tmdb_token_set"], json!(true)); + assert!(r.get("tmdb_token").is_none()); + assert_eq!(r["resolve_torrent_files"], json!(true)); + assert_eq!(r["indexers"][0]["name"], json!("prowlarr")); + assert_eq!(r["indexers"][0]["api_key_set"], json!(true)); + assert!(r["indexers"][0].get("api_key").is_none()); + } + + #[test] + fn merge_blank_secret_keeps_base() { + let base = json!({"tmdb_token": "keepme"}); + let inc = json!({"tmdb_token": ""}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("keepme")); + } + + #[test] + fn merge_nonblank_secret_overwrites() { + let base = json!({"tmdb_token": "old"}); + let inc = json!({"tmdb_token": "new"}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("new")); + } + + #[test] + fn merge_record_array_backfills_blank_secret_by_identifier() { + let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]}); + // UI re-submits the indexer with a blank api_key → keep the stored one. + let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["indexers"][0]["api_key"], json!("secret")); + assert_eq!(m["indexers"][0]["name"], json!("prowlarr")); + } + + #[test] + fn merge_bool_overwrites_when_present() { + let base = json!({"resolve_torrent_files": true}); + let inc = json!({"resolve_torrent_files": false}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["resolve_torrent_files"], json!(false)); + } +} diff --git a/src/config_page.html b/src/config_page.html new file mode 100644 index 0000000..17d9d74 --- /dev/null +++ b/src/config_page.html @@ -0,0 +1,268 @@ + + + + + +feeder config + + + +

Plugin configuration

+

Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked * are required.

+ +
+ + + + + diff --git a/src/enrich.rs b/src/enrich.rs new file mode 100644 index 0000000..b55a9f8 --- /dev/null +++ b/src/enrich.rs @@ -0,0 +1,614 @@ +//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline. +//! +//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base +//! record, then drives the **reused meta-sort enrichment plugins** +//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each +//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to +//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches +//! enrichment — full separation of concerns. +//! +//! ## Ordering — why we poll +//! +//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from +//! the record), and those are produced by filename-parser. So the pipeline is +//! **filename-parser → tmdb**. The plugin `/process` contract is *async* +//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after +//! kicking filename-parser we **poll meta-core** until `originalTitle` appears +//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps +//! the feeder stateless — no callback-correlation bookkeeping. The plugins' +//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`. +//! +//! The whole pipeline runs in a spawned background task so it never adds latency +//! to the libp2p resolve path; the meta-share ingester propagates the enriched +//! record on its next poll (~eventually consistent). + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use tracing::{debug, warn}; + +use crate::meta_core::{self, Provenance}; +use crate::types::DiscoveryRecord; + +/// Operator-supplied enrichment wiring, parsed once from the feeder env. +#[derive(Clone, Debug)] +pub struct EnrichmentConfig { + /// meta-core root the feeder self-publishes to + hands to the plugins. + pub meta_core_url: String, + /// `metamesh-plugin-filename-parser` base URL (optional). + pub filename_parser_url: Option, + /// `metamesh-plugin-tmdb` base URL (optional). + pub tmdb_url: Option, + /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`. + pub tmdb_token: Option, + /// TMDB metadata language (default `en-US`). + pub tmdb_language: String, + /// Stable per-deployment id stamped into `provenance.gatewayPeer`. + pub gateway_peer: String, + /// URL the plugins POST their completion callback to (the feeder's own + /// no-op `/enrich/callback`). Polling is the real completion signal; this + /// only keeps the plugins from logging a failed-callback warning. + pub callback_url: String, + /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the + /// pipeline drives subtitle fetching after tmdb resolves the ids — the + /// single reference implementation of OpenSubtitles fetching (the gateway + /// no longer carries its own). + pub opensubtitles_url: Option, + /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`. + pub opensubtitles_api_key: Option, + /// OpenSubtitles account username (required by the plugin for downloads). + pub opensubtitles_username: Option, + /// OpenSubtitles account password. + pub opensubtitles_password: Option, + /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). + pub opensubtitles_languages: String, +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the +/// app-compose convention `-feeder` → `-:8080` +/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets +/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit +/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns +/// `None` when `HOSTNAME` is unset or not in `-feeder` form, in which case +/// the caller falls back to the env seed (then soft-skips that plugin). +fn derive_sidecar_url(plugin: &str) -> Option { + let host = env_nonempty("HOSTNAME")?; + let prefix = host.strip_suffix("-feeder")?; + Some(format!("http://{prefix}-{plugin}:8080")) +} + +impl EnrichmentConfig { + /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset — + /// the **bundled-feeder** path (records returned over HTTP, the gateway + /// dispatcher stores them). Self-publishing feeders that source meta-core from + /// their own dashboard config use [`from_meta_core`](Self::from_meta_core). + pub fn from_env() -> Option { + Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?)) + } + + /// Build with an explicit meta-core root — sourced from the feeder's own + /// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The + /// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the + /// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest + /// keep their env seeds / defaults. Secrets (tmdb token, language) are + /// overlaid by the caller from the per-feeder config. + pub fn from_meta_core(meta_core_url: String) -> Self { + EnrichmentConfig { + meta_core_url, + filename_parser_url: env_nonempty("FILENAME_PARSER_URL") + .or_else(|| derive_sidecar_url("filename-parser")), + tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")), + tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")), + tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()), + gateway_peer: env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()), + callback_url: env_nonempty("META_FEEDER_CALLBACK_URL") + .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()), + opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"), + opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"), + opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"), + opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), + opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") + .unwrap_or_else(|| "en".to_string()), + } + } +} + +/// How long to wait for filename-parser to merge `originalTitle` before handing +/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous +/// ceiling, not an expected wait. +const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5); +/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the +/// sibling files. tmdb hits the network, so a longer ceiling than filename. +const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Show-level TMDB fields that are **fanned** from the primary file to every +/// sibling file in a multi-file (season-pack) torrent — they're identical for +/// every episode of a show, so we resolve tmdb **once** and copy these across +/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which +/// gateway BT records don't carry, so without this each file would re-hit TMDB). +/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`, +/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file. +const ANCHOR_EXACT: &[&str] = &[ + "tmdbid", + "imdbid", + "poster", + "backdrop", + "rating", + "releasedate", + "studio", + "genres", + "tags", +]; +const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"]; + +/// One file (or pack) to enrich: its CID, the filename to parse, and the base +/// fields already written to meta-core (handed to the plugins as `existingMeta`). +#[derive(Clone, Debug)] +pub struct EnrichTarget { + pub cid: String, + pub file_name: String, + pub base: BTreeMap, +} + +/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to +/// clone (just a reqwest client + an `Arc`), so it can be moved into a +/// spawned per-resolve task. +#[derive(Clone)] +pub struct Enricher { + http: reqwest::Client, + cfg: Arc, +} + +#[derive(Serialize)] +struct ProcessRequest<'a> { + #[serde(rename = "taskId")] + task_id: String, + cid: &'a str, + #[serde(rename = "filePath")] + file_path: &'a str, + #[serde(rename = "callbackUrl")] + callback_url: &'a str, + #[serde(rename = "metaCoreUrl")] + meta_core_url: &'a str, + #[serde(rename = "existingMeta")] + existing_meta: &'a BTreeMap, +} + +impl Enricher { + pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self { + Enricher { + http, + cfg: Arc::new(cfg), + } + } + + pub fn meta_core_url(&self) -> &str { + &self.cfg.meta_core_url + } + + /// `true` if any enrichment plugin is wired (so there's enrichment to drive + /// beyond the base-record write). + pub fn has_plugins(&self) -> bool { + self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some() + } + + /// Write the feeder's base record into meta-core (merge/upsert) under `cid`. + /// Identical shape to the gateway dispatcher's write (provenance + the + /// `cids/` key-set collapse). + pub async fn write_base_record( + &self, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now( + &record.upstream_id, + &record.record_id, + &self.cfg.gateway_peer, + ); + let body = meta_core::build_metadata_body(record, &prov); + meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await + } + + /// Run the video enrichment pipeline for one resolved CID: filename-parser + /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each + /// plugin writes its own fields to meta-core. Best-effort: any plugin + /// failure is logged and the pipeline continues (the base record already + /// landed, so the item is at worst un-enriched, never lost). + pub async fn enrich_video( + &self, + cid: &str, + file_name: &str, + base_fields: BTreeMap, + ) { + let mut meta = base_fields; + + // 1. filename-parser → originalTitle / movieYear / videoType / season / episode. + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await { + Ok(()) => { + // Poll for the merged fields so tmdb gets the clean title. + if let Some(merged) = self + .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + { + meta = merged; + } else { + debug!( + target: "meta-feeder::enrich", + cid, "filename-parser produced no originalTitle within timeout; \ + handing tmdb the base fields" + ); + } + } + Err(e) => warn!( + target: "meta-feeder::enrich", + cid, error = %e, "filename-parser /process failed (non-fatal)" + ), + } + } + + // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video + + // a clean originalTitle, both now present on `meta`). + if let Some(tmdb_url) = self.cfg.tmdb_url.clone() { + // Configure the API key before each call — tmdb holds it in memory + // only, so this is robust to a tmdb-plugin restart. Cheap (local). + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /process failed (non-fatal)" + ); + } + } + + // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which + // tmdb writes async). Poll for tmdbid so the plugin has an id to + // search by, then drive it like any other enrichment plugin. + if self.cfg.opensubtitles_url.is_some() { + let merged = self + .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or(meta); + self.enrich_subtitles(cid, file_name, &merged).await; + } + } + + /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files + + /// an optional whole-torrent pack record. filename-parser runs on every + /// target; tmdb runs **once** on the primary video file and its show-level + /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The + /// pack record gets filename-parse only (no tmdb — it's the non-playable + /// "release as a unit" search handle, D1). Best-effort throughout. + pub async fn enrich_bundle(&self, videos: Vec, pack: Option) { + // Pack: filename-parse only, no tmdb. + if let Some(p) = pack { + self.fire_filename(&p).await; + } + + if videos.is_empty() { + return; + } + if videos.len() == 1 { + // Degenerate single-video bundle → the plain pipeline (filename → tmdb). + let v = &videos[0]; + self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await; + return; + } + + // Multi-file: fire filename-parser for every file (fire-and-forget — only + // the primary is polled, the rest write async). + for v in &videos { + self.fire_filename(v).await; + } + + // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen). + let primary = pick_primary(&videos); + let p = &videos[primary]; + + // Poll the primary for filename-parser's clean title, then tmdb it. + let merged = self + .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + .unwrap_or_else(|| p.base.clone()); + + let Some(tmdb_url) = self.cfg.tmdb_url.clone() else { + return; // no tmdb wired → filename-only enrichment, done. + }; + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed"); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed"); + return; + } + + // Read the resolved anchor back off the primary, then fan it to siblings. + let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await + else { + debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan"); + return; + }; + let anchor = extract_anchor(&anchor_rec); + if anchor.is_empty() { + return; + } + for (i, v) in videos.iter().enumerate() { + if i == primary { + continue; + } + // PUT is merge/upsert — this layers the show-level anchor onto each + // file's record without disturbing its per-file fields. + if let Err(e) = + meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await + { + warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); + } + } + + // Subtitles per video: each episode needs its own (its season/episode + // differs), but they share the fanned tmdbid. Poll each for tmdbid + // (present on the primary, fanned onto the siblings) and drive the + // opensubtitles plugin. Best-effort + quota-guarded inside the plugin. + if self.cfg.opensubtitles_url.is_some() { + for v in &videos { + let merged = self + .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or_else(|| v.base.clone()); + self.enrich_subtitles(&v.cid, &v.file_name, &merged).await; + } + } + } + + /// Fire filename-parser `/process` for one target (fire-and-forget; the + /// plugin merges its fields to meta-core async). + async fn fire_filename(&self, t: &EnrichTarget) { + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + if let Err(e) = self + .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base) + .await + { + warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed"); + } + } + } + + /// POST `/process` to a plugin. Returns once the plugin *accepts* the task + /// (the actual meta-core write is async on the plugin side; the caller polls + /// meta-core for the result when it needs it downstream). + async fn process( + &self, + plugin_url: &str, + plugin: &str, + cid: &str, + file_path: &str, + existing_meta: &BTreeMap, + ) -> anyhow::Result<()> { + let url = format!("{}/process", plugin_url.trim_end_matches('/')); + let body = ProcessRequest { + task_id: format!("{plugin}-{cid}-{}", now_nanos()), + cid, + file_path, + callback_url: &self.cfg.callback_url, + meta_core_url: &self.cfg.meta_core_url, + existing_meta, + }; + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("{plugin} POST {url} returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the tmdb plugin with the API key + language. + async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> { + let Some(token) = self.cfg.tmdb_token.clone() else { + anyhow::bail!("no TMDB token configured"); + }; + let url = format!("{}/configure", tmdb_url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { "apiKey": token, "language": self.cfg.tmdb_language } + }); + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("tmdb /configure returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the opensubtitles plugin with the API key, account + /// credentials, and wanted languages. + async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> { + let Some(key) = self.cfg.opensubtitles_api_key.clone() else { + anyhow::bail!("no OpenSubtitles API key configured"); + }; + let cfg_url = format!("{}/configure", url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { + "apiKey": key, + "username": self.cfg.opensubtitles_username, + "password": self.cfg.opensubtitles_password, + "languages": self.cfg.opensubtitles_languages, + } + }); + let resp = self.http.post(&cfg_url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("opensubtitles /configure returned {status}: {text}"); + } + Ok(()) + } + + /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or + /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta` + /// and writes the `subtitles//` key-set straight to meta-core). + /// Best-effort: any failure is logged and swallowed. + async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) { + let Some(url) = self.cfg.opensubtitles_url.clone() else { + return; + }; + if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") { + debug!( + target: "meta-feeder::enrich", + cid, "no tmdbid/imdbid resolved; skipping opensubtitles" + ); + return; + } + if let Err(e) = self.configure_opensubtitles(&url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /process failed (non-fatal)" + ); + } + } + + /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout` + /// elapses; returns the merged field map on success. + async fn poll_until_field( + &self, + cid: &str, + field: &str, + timeout: Duration, + ) -> Option> { + let deadline = SystemTime::now() + timeout; + loop { + match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await { + Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => { + return Some(fields); + } + Ok(_) => {} + Err(e) => debug!( + target: "meta-feeder::enrich", + cid, error = %e, "poll get_record failed" + ), + } + if SystemTime::now() >= deadline { + return None; + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } +} + +fn now_nanos() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// Pick the primary video file index: largest by `sizeByte`, ties broken toward +/// the first (lowest index). Falls back to 0 for an all-unsized bundle. +fn pick_primary(videos: &[EnrichTarget]) -> usize { + videos + .iter() + .enumerate() + .max_by_key(|(i, v)| { + let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0); + // larger size wins; on a tie the smaller index wins (Reverse). + (size, std::cmp::Reverse(*i)) + }) + .map(|(i, _)| i) + .unwrap_or(0) +} + +/// Project a resolved record down to the show-level anchor fields that are safe +/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]). +fn extract_anchor(fields: &BTreeMap) -> BTreeMap { + fields + .iter() + .filter(|(k, _)| { + ANCHOR_EXACT.contains(&k.as_str()) + || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p)) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_env_requires_meta_core_url() { + // Not set in the test env → None (legacy gateway-stores path). + // (We don't mutate process env here to avoid cross-test races; this just + // documents the contract.) + assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some()); + } + + fn tgt(cid: &str, size: u64) -> EnrichTarget { + EnrichTarget { + cid: cid.into(), + file_name: cid.into(), + base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]), + } + } + + #[test] + fn pick_primary_is_largest_then_first() { + let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)]; + // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins. + assert_eq!(pick_primary(&v), 1); + } + + #[test] + fn pick_primary_all_unsized_defaults_first() { + let v = vec![ + EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() }, + EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() }, + ]; + assert_eq!(pick_primary(&v), 0); + } + + #[test] + fn extract_anchor_keeps_show_level_drops_per_file() { + let rec = BTreeMap::from_iter([ + ("tmdbid".to_string(), "10378".to_string()), + ("imdbid".to_string(), "tt1254207".to_string()), + ("poster".to_string(), "bafkPOSTER".to_string()), + ("genres".to_string(), "Animation|Comedy".to_string()), + ("plot/eng".to_string(), "a plot".to_string()), + ("tags".to_string(), "tmdb-verified".to_string()), + // per-file — must NOT be fanned: + ("originalTitle".to_string(), "Big Buck Bunny".to_string()), + ("season".to_string(), "1".to_string()), + ("episode".to_string(), "3".to_string()), + ("fileName".to_string(), "ep3.mkv".to_string()), + ]); + let a = extract_anchor(&rec); + assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng")); + assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid")); + assert!(!a.contains_key("originalTitle"), "originalTitle is per-file"); + assert!(!a.contains_key("season") && !a.contains_key("episode")); + assert!(!a.contains_key("fileName")); + } +} diff --git a/src/filename_meta.rs b/src/filename_meta.rs new file mode 100644 index 0000000..8c93c11 --- /dev/null +++ b/src/filename_meta.rs @@ -0,0 +1,1309 @@ +//! Season / episode extraction from a torrent release title. +//! +//! This is a Rust port of `@metazla/filename-tools` +//! (`packages/filename-tool/src/lib/...`) — specifically the +//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` → +//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the +//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the +//! best season/episode heuristic we have (the meta-sort `torrent` plugin +//! already uses it), so rather than invent a second, weaker heuristic for +//! the gateway we mirror its pattern set and ordering verbatim. +//! +//! ## Differences from the TS original (deliberate, documented) +//! +//! 1. **No parent-folder fallback.** The TS extractor also reads +//! `Season N` out of the *parent directory* name. A torznab hit is a +//! single release title with no directory context, so that branch is +//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup +//! simply has nothing to read. +//! 2. **Rightmost-number fallback uses the captured digits, not the raw +//! regex match.** The TS `extractRightmostNumber` does +//! `parseFloat(fullMatch)` where the match can include a leading `.` +//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat` +//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the +//! bare number) instead, which is what the function *means* to return. +//! +//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so +//! a movie's release year can't be misread as an episode number by the +//! rightmost-number fallback. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; + +/// Structured season/episode extracted from a release title. Each field +/// is the stringified number exactly as the TS extractor would emit it +/// (`"" + n`): whole numbers render without a decimal, half-episode +/// specials keep the `.5` (e.g. `"13"`, `"13.5"`). +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SeasonEpisode { + pub season: Option, + pub episode: Option, + /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`). + /// Set together with [`episode_end`]; mutually exclusive with the + /// single-episode [`episode`] field. + pub episode_start: Option, + /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`). + pub episode_end: Option, + /// `season * 10000 + episode` disambiguator (matches the TS + /// `increment` field). Computed for completeness; the torznab caller + /// currently only persists `season`/`episode`. + pub increment: Option, + /// True when this release is a **pack/batch** rather than a single + /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season + /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with + /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack + /// is deliberately **not** given a single `episode`. + pub is_pack: bool, + /// True when [`season`] came from an explicit season-bearing token + /// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left + /// season-less (no "assume season 1" default), so a per-file reparse uses + /// this to know it should borrow the season from the pack/parent title (see + /// [`extract_season_episode_with_context`]). + pub season_explicit: bool, +} + +impl SeasonEpisode { + /// A pack carrying a concrete inclusive episode range. + pub fn has_episode_range(&self) -> bool { + self.episode_start.is_some() && self.episode_end.is_some() + } +} + +// --------------------------------------------------------------------------- +// Pattern fragments — copied 1:1 from +// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`. +// --------------------------------------------------------------------------- + +const SW: &str = r"(?:\.|\b)"; // a dot or word boundary +const S: &str = r"(?:\.|\b|\s)"; +const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored +const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5) +const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5) +const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5) +const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99 +const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5) + +/// Season-AND-episode patterns, longest/most-specific first. Each has +/// exactly two capture groups: group 1 = season, group 2 = episode. +/// Order matters — the first to match wins (mirrors the TS array order). +static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"), + format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"), + format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"), + format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"), + format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season+episode pattern compiles")) + .collect() +}); + +/// Episode-only patterns. Each has exactly one capture group: the episode. +static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"), + format!(r"(?i){SW}E{S}*{EP4MY}{END}"), + format!(r"(?i){SW}Ep{EP4MY}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("episode-only pattern compiles")) + .collect() +}); + +/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number. +static SOLO_EP: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles")); + +// --------------------------------------------------------------------------- +// Episode-range / pack detection. Anime + TV batches encode a *range* of +// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season +// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so +// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a +// batch collapses to one phantom episode. We detect ranges FIRST (before +// `find_season_and_episode`) and emit `episode_start`/`episode_end` + +// `is_pack`, so the caller can stamp `contentKind=pack` instead. +// +// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`: +// - ranges use a `-`/`~` separator with TWO numbers and `end > start`; +// - the bare form requires the start digit to be preceded by a space/`_`/`-` +// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read +// as a range start — that release falls through to the single-episode path; +// - 4-digit year-like / out-of-range operands are rejected. +// --------------------------------------------------------------------------- + +/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`. +static RANGE_BRACKET: LazyLock = LazyLock::new(|| { + Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles") +}); + +/// CJK episode range: `第29-38話` / `第29~38話`. +static RANGE_CJK: LazyLock = LazyLock::new(|| { + Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles") +}); + +/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces +/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a +/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight +/// separator is the disambiguator from the very common `Show 2 - 11` / +/// `Show - 11` episode-separator convention (spaced dash → single episode, not +/// a range); spaced ranges only come through the unambiguous bracketed form. +static RANGE_BARE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)") + .expect("bare range compiles") +}); + +/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single +/// season. Matched only as a fallback when no episode range is present. +static SEASON_RANGE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles") +}); + +/// `(start, end)` is a plausible inclusive episode range: ascending, and +/// neither operand is a 4-digit year / resolution-like number. +fn plausible_range(start: u32, end: u32) -> bool { + end > start && (1..1900).contains(&start) && end < 1900 +} + +/// Detect an episode range anywhere in the (precleaned) title. Returns the +/// first plausible `(start, end)` found, trying the unambiguous bracketed and +/// CJK forms before the bare form. +fn detect_episode_range(s: &str) -> Option<(u32, u32)> { + for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] { + for caps in re.captures_iter(s) { + let start = caps.get(1).and_then(|m| m.as_str().parse::().ok()); + let end = caps.get(2).and_then(|m| m.as_str().parse::().ok()); + if let (Some(a), Some(b)) = (start, end) { + if plausible_range(a, b) { + return Some((a, b)); + } + } + } + } + None +} + +/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix. +static SFV: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles")); + +// --------------------------------------------------------------------------- +// Keyword pre-clean — copied from +// `filename-tool/src/lib/config/KeywordsArray.ts`. +// Removed before season/episode detection so release tags like `x264`, +// `1080p`, `720` can't be mistaken for episode numbers by the fallback. +// --------------------------------------------------------------------------- + +/// Word-bounded keywords (resolution/codec/source/group tags). Order is +/// irrelevant once compiled individually, but we sort longest-first to +/// match the TS constructor (`keyword.length` desc) for identical +/// overlap behaviour. +const KEYWORDS: &[&str] = &[ + "VF", + "aXXo", + "STFR", + "SUBFORCED", + "BrRipx264", + "YIFY", + "TRUEFRENCH", + "VFF", + "ATVP", + "RenewalEX", + "BD", + "DD", + "8bits", + "8 bits", + "10bits", + "10 bits", + "part 1", + "part 2", + "part 3", + "part 4", + "part A", + "part B", + "part C", + "part D", + "AAC5", + "AAC5.1", + "1080i", + "1080p", + "2160p", + "480i", + "480p", + "4k", + "576i", + "576p", + "720", + "720i", + "720p", + "aac", + "aac4", + "ac3", + "amzn", + "apple tv+", + "avi", + "bbc", + "bd5", + "bdrip", + "blu-ray", + "bluray", + "brrip", + "cam", + "cw", + "dc", + "dcu", + "ddp5.1", + "director's cut", + "disney+", + "divx", + "divx5", + "dl", + "dsr", + "dsrip", + "dts", + "dual audio", + "dubbed", + "dvd", + "dvdivx", + "dvdr", + "dvdrip", + "dvdscr", + "dvdscreener", + "eng", + "eng sub", + "esp", + "fan edit", + "fhd", + "flv", + "fs", + "ger", + "german", + "h.264", + "h.265", + "h264", + "h265", + "hardcoded", + "hbo", + "hd", + "hddvd", + "hdr", + "hdrip", + "hdtv", + "hdtvrip", + "hevc", + "hq", + "hrhd", + "hrhdtv", + "hulu", + "imax", + "ita", + "jpn", + "ld", + "md", + "mkv", + "mp3", + "mp4", + "mpeg", + "mpg", + "mq", + "multi", + "multisubs", + "netflix", + "nf", + "nfofix", + "ntsc", + "ogg", + "ogm", + "ova", + "pal", + "pdtv", + "r3", + "r5", + "rerip", + "rsvcd", + "screener", + "sd", + "se", + "subbed", + "svcd", + "tc", + "telecine", + "telesync", + "ts", + "tv series", + "uhd", + "uhdtv", + "uhdv", + "v2", + "vcd", + "vostfr", + "web", + "web-dl", + "webcast", + "webrip", + "wmv", + "ws", + "www", + "x264", + "x265", + "xsvcd", + "xvid", + "xvidvd", + "xxx", + "bits", + "AnimeServ", +]; + +/// Substrings removed without word boundaries (the TS `substringArray`). +const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"]; + +/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first. +static KEYWORD_RES: LazyLock> = LazyLock::new(|| { + let mut kws: Vec<&str> = KEYWORDS.to_vec(); + kws.sort_by_key(|k| std::cmp::Reverse(k.len())); + kws.into_iter() + .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles")) + .collect() +}); + +/// Compiled substring regexes (no boundaries, case-insensitive), longest-first. +static SUBSTRING_RES: LazyLock> = LazyLock::new(|| { + let mut subs: Vec<&str> = SUBSTRINGS.to_vec(); + subs.sort_by_key(|s| std::cmp::Reverse(s.len())); + subs.into_iter() + .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles")) + .collect() +}); + +// --------------------------------------------------------------------------- +// Language tags — anime/torznab upstreams encode language as release-name +// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a +// structured field. `extract_languages` reads *every* tag off the raw title +// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`. +// The gateway writes each as a `languages/ = "true"` key-set member +// (METADATA_KEYS §9) that the preferred-language filter matches against. +// +// Patterns are ported from the battle-tested torrent-name parsers +// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead +// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`, +// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex` +// crate has no lookaround. Two defensive choices do most of the work: +// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They +// collide with ordinary words, names, and domain TLDs. We rely on the +// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead. +// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside +// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`. +// --------------------------------------------------------------------------- + +/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD +/// or path token can't be read as a language. Only `www.`-prefixed runs are +/// removed — a generic `host.tld` matcher would eat dotted release names like +/// `Spy.x.Family`. Runs on the raw (still-dotted) title. +static URL_STRIP: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles")); + +/// Guard phrases removed from the separator-normalized title before language +/// matching, because each contains a substring that would otherwise +/// false-positive a tag. None of these phrases is itself a language tag, so +/// removal is loss-free for detection — this is how we reproduce the upstream +/// parsers' lookbehind guards without lookaround: +/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese) +/// - `tel aviv` → guards `\btel\b` (Telugu) +/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker) +static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b") + .expect("lang guard re compiles") +}); + +/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match +/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js` +/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`), +/// because meta-share's "languages I read" preference is the source of truth +/// the meta-watch filter compares against — the enriched value must be in the +/// same space the chips use. (This deliberately diverges from `gutenberg.rs` / +/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.) +/// +/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for +/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release +/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul` +/// and `fre`: every pattern that matches contributes its code, so the result +/// is a set, not a first-wins pick. +static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| { + [ + // Multi / dual-audio → "mul". `dl` matches only after "web dl" has + // been stripped (see LANG_GUARD_STRIP). + ( + r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b", + "mul", + ), + // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…). + ( + r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b", + "fre", + ), + (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"), + (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"), + (r"\bitalian\b|\bita\b", "ita"), + (r"\bgerman\b|\bger\b|\bdeu\b", "ger"), + ( + r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b", + "spa", + ), + (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"), + (r"\bkorean\b|\bkor\b", "kor"), + ( + r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b", + "chi", + ), + (r"\brussian\b|\brus\b", "rus"), + (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"), + (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"), + (r"\bczech\b|\bcze\b|\bces\b", "cze"), + (r"\bslovak\b|\bslo\b|\bslk\b", "slo"), + (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"), + ( + r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b", + "rum", + ), + (r"\bgreek\b|\bgre\b|\bell\b", "gre"), + (r"\bukrainian\b|\bukr\b", "ukr"), + (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"), + (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"), + (r"\bnorwegian\b|\bnorsk\b", "nor"), + (r"\bdanish\b|\bdansk\b", "dan"), + (r"\bfinnish\b|\bsuomi\b", "fin"), + (r"\bturkish\b|\btur\b", "tur"), + (r"\barabic\b|\bara\b", "ara"), + (r"\bhindi\b|\bhin\b", "hin"), + (r"\btamil\b|\btam\b", "tam"), + (r"\btelugu\b|\btel\b", "tel"), + (r"\bmalayalam\b|\bmal\b", "mal"), + (r"\bvietnamese\b|\bvie\b", "vie"), + (r"\bthai\b|\btha\b", "tha"), + ] + .iter() + .map(|(p, code)| { + ( + Regex::new(&format!("(?i){p}")).expect("language tag re compiles"), + *code, + ) + }) + .collect() +}); + +/// Characters `cleanSpecialChars` replaces with a space (note: `.` and +/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07` +/// structure). +const SPECIAL_CHARS: &[char] = &[ + '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"', + '`', '~', +]; + +/// Extract season/episode from a release title. +/// +/// Faithful to `computeSeasonAndEpisodeFromFilename` → +/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath` +/// (minus the parent-folder branch — see the module doc). +pub fn extract_season_episode(title: &str) -> SeasonEpisode { + let cleaned = preclean(title); + // Range detection runs on the *raw* title (with `~` normalized to `-`), + // not the precleaned one: `preclean` replaces `~` with a space (it's a + // "special char"), which would erase a `26~37` range, and it never touches + // the `(`/`[` brackets the bracketed form anchors on. + let range_src = title.replace('~', "-"); + + // Pack detection runs first — an episode range (`(01-10)`) would otherwise + // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}` + // pattern. A pack is never given a single `episode`. + if let Some((start, end)) = detect_episode_range(&range_src) { + let season = find_pack_season(&cleaned); + return SeasonEpisode { + is_pack: true, + episode_start: Some(start.to_string()), + episode_end: Some(end.to_string()), + // An explicit season token (`S2 (01-10)`) buckets the pack; a bare + // range (`Frieren (01-28)`) leaves the season for the caller. + season: season.map(|s| s.to_string()), + season_explicit: season.is_some(), + ..Default::default() + }; + } + // Season range (`S01-S03`) — a whole-series pack, no single season/episode. + if SEASON_RANGE.is_match(&range_src) { + return SeasonEpisode { + is_pack: true, + ..Default::default() + }; + } + + let (sea, ep) = find_season_and_episode(&cleaned); + + let mut out = SeasonEpisode::default(); + // season/episode 0 are valid; only < 0 means "not found". + if sea >= 0.0 { + out.season = Some(fmt_num(sea)); + out.season_explicit = true; + } + if ep >= 0.0 { + out.episode = Some(fmt_num(ep)); + // An episode with no season is left season-LESS on purpose: a season is + // only asserted when explicitly hinted — an S-token in the title here, + // or (via extract_season_episode_with_context) the parent/pack title. + // Absolute-numbered releases ("Show - 28") thus carry an episode and no + // season, so the UI buckets them under "Episodes" rather than a + // fabricated "Season 1". `increment` encodes a season, so it is only + // emitted when the season is known. + if sea >= 0.0 { + out.increment = Some(fmt_num(sea * 10000.0 + ep)); + } + } else if sea >= 0.0 { + // Season present, no episode → a whole-season pack (`Season 3`, `S03`). + out.is_pack = true; + } + out +} + +/// Extract season/episode using a `/` style context — the +/// per-file path of an opened multi-file (pack) torrent. Parses the file's own +/// `filename` first; when the filename yields an episode but **no** season, the +/// season is recovered from the `parent` (the pack/torrent title). This +/// restores the parent-folder season fallback the TS +/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped +/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves +/// to S2E3 rather than the inherited pack-level number. +pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode { + let mut file_se = extract_season_episode(filename); + // Only borrow the parent's season for a concrete single episode whose own + // filename carries no *explicit* season token (so it is otherwise + // season-less). A per-file record is an episode, not a pack, so we don't + // propagate the parent's pack/range shape here. + if file_se.episode.is_some() && !file_se.season_explicit { + if let Some(s) = find_pack_season(&preclean(parent)) { + file_se.season = Some(s.to_string()); + file_se.season_explicit = true; + if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) { + file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep)); + } + } + } + file_se +} + +/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`, +/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to +/// recover a per-file season from the parent. Excludes the bare-number season +/// form so an episode-only title isn't mis-bucketed. +fn find_pack_season(s: &str) -> Option { + let padded = format!(" {s} "); + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let v = parse_group(&caps, 1); + if v >= 0.0 { + return Some(v as i64); + } + } + } + None +} + +/// Best-effort set of languages a release supports (audio **and** subtitles, +/// mixed), inferred from its title tags. +/// +/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`, +/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we +/// read them off the raw title here. Returns the **full set** of ISO 639-2/B +/// alpha-3 codes found (matching meta-share's UI vocabulary — see +/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is +/// empty when the title carries no recognizable tag — the caller turns that +/// into the `languages/und` sentinel. +pub fn extract_languages(title: &str) -> BTreeSet { + // 1. Strip `www.…` domain noise while the dots are still intact. + let stripped = URL_STRIP.replace_all(title, " "); + // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags + // anchor inside dotted/underscored/hyphenated release names — and so + // the guard phrases ("web dl", "shang chi") match regardless of which + // separator the original used. + let norm: String = stripped + .chars() + .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c }) + .collect(); + // 3. Remove guard phrases that would otherwise false-positive a tag. + let guarded = LANG_GUARD_STRIP.replace_all(&norm, " "); + // 4. Collect every matching code (a set — dedupes, deterministic order). + let mut out = BTreeSet::new(); + for (re, code) in LANGUAGE_TAGS.iter() { + if re.is_match(&guarded) { + out.insert((*code).to_string()); + } + } + out +} + +// --------------------------------------------------------------------------- +// Quality — the resolution *tier* of a release, normalized to a single label +// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured +// resolution, so — like season/episode and language — the release title is the +// only source. Mirrors the client-side `parseVersion()` heuristic in +// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's +// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to +// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height. +// --------------------------------------------------------------------------- + +/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH` +/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title +/// carries no recognizable resolution. +pub fn extract_quality(title: &str) -> Option { + static TIER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles") + }); + static WXH: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles")); + // Normalize `.`/`_` separators so `\b`-anchored tokens match inside + // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`). + let t = title.replace(['.', '_'], " "); + if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) { + let tok = m.as_str().to_lowercase(); + return Some( + match tok.as_str() { + "4k" => "2160p", + "1080i" => "1080p", + "720i" => "720p", + other => other, + } + .to_string(), + ); + } + if let Some(h) = WXH + .captures(&t) + .and_then(|c| c.get(1)) + .and_then(|m| m.as_str().parse::().ok()) + { + let tier = if h >= 2000 { + "2160p" + } else if h >= 1000 { + "1080p" + } else if h >= 700 { + "720p" + } else if h >= 500 { + "576p" + } else { + "480p" + }; + return Some(tier.to_string()); + } + None +} + +/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12 +/// structured search facet). Title-derived / best-effort — same provenance and +/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in +/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens +/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and +/// any stream-derived value. +pub fn extract_codec(title: &str) -> Option { + static CODEC: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b") + .expect("codec re compiles") + }); + // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored + // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`). + let t = title.replace(['.', '_'], " "); + let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase(); + let cleaned = tok.replace(' ', ""); + let norm = match cleaned.as_str() { + "x265" | "h265" | "hevc" => "hevc", + "x264" | "h264" | "avc" => "h264", + other => other, // av1, xvid, divx, vp9 + }; + Some(norm.to_string()) +} + +/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + +/// `cleanSpace`, in that order. +fn preclean(filename: &str) -> String { + let mut s = filename.to_string(); + for re in KEYWORD_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + // removeSFV — first occurrence only (TS uses a non-global replace). + s = SFV.replace(&s, "").into_owned(); + // cleanSpecialChars — replace each special char with a space. + s = s + .chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect(); + // cleanSpace — collapse runs of whitespace and trim. + s.split_whitespace().collect::>().join(" ") +} + +/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0` +/// for "not found". +fn find_season_and_episode(filename: &str) -> (f64, f64) { + // Pad with spaces so word-boundary fragments anchor at the ends + // exactly as the TS code does (`" " + filename + " "`). + let padded = format!(" {filename} "); + + for re in SEASON_EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + let episode = parse_group(&caps, 2); + return (season, episode); + } + } + for re in EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let episode = parse_group(&caps, 1); + return (-1.0, episode); + } + } + // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught + // here — *before* the bare rightmost-number fallback — so a whole-season + // pack reports `(season, no-episode)` instead of the fallback mis-reading + // its season digit as an episode and `extract_season_episode` then + // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03` + // bug). The TS pipeline relied on a parent-folder lookup for this case, + // which the gateway dropped (see module doc). + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + if season >= 0.0 { + return (season, -1.0); + } + } + } + // Last hope: the rightmost bare number. `if (number)` in the TS is + // falsy for 0, so a lone 0 doesn't count as an episode. + if let Some(n) = rightmost_number(&padded) { + if n != 0.0 { + return (-1.0, n); + } + } + (-1.0, -1.0) +} + +fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 { + caps.get(idx) + .and_then(|m| m.as_str().parse::().ok()) + .unwrap_or(-1.0) +} + +/// `extractRightmostNumber` — the last `soloEp` match's number. +fn rightmost_number(s: &str) -> Option { + SOLO_EP + .captures_iter(s) + .filter_map(|c| c.get(1)) + .last() + .and_then(|m| m.as_str().parse::().ok()) +} + +/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64 +/// `Display` already drops the trailing `.0`, so this is a thin wrapper +/// kept for intent. +fn fmt_num(n: f64) -> String { + format!("{n}") +} + +// --------------------------------------------------------------------------- +// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` + +// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`. +// The gateway previously used a separate hand-rolled `clean_torrent_title` +// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and +// season-word/episode-range stripping — so TMDB enrichment failed on many +// release-name shapes. Reuse the already-ported building blocks instead. +// --------------------------------------------------------------------------- + +/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` / +/// `S N` / trailing bare number and everything after it. +static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + format!(r"(?i){SW}{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-only pattern compiles")) + .collect() +}); + +/// Season-only patterns restricted to the **keyword'd** forms (`Season N`, +/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a +/// whole-season *pack* (no episode) before the bare rightmost-number fallback +/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}` +/// entry: that one matches any lone number and would turn an episode-only +/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because +/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`. +static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-word-only pattern compiles")) + .collect() +}); + +/// `cleanTags` — bracketed/parenthesised/braced groups. +static TAG_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles")); +/// `removeTagSeparator` — stray bracket chars. +static TAG_SEP_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles")); +/// `removeYear` — a standalone 19xx/20xx. +static YEAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles")); +/// `removeBeyondHyphen` — capture everything before the first ` - ` +/// (space-delimited) hyphen. +static BEYOND_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles")); +static TRAILING_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles")); + +fn clean_special_chars(s: &str) -> String { + s.chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect() +} + +fn remove_keywords(s: &str) -> String { + let mut out = s.to_string(); + for re in KEYWORD_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + out +} + +fn remove_beyond_hyphen(s: &str) -> String { + if let Some(c) = BEYOND_HYPHEN_RE.captures(s) { + return c + .get(1) + .map(|m| m.as_str().trim().to_string()) + .unwrap_or_default(); + } + TRAILING_HYPHEN_RE.replace(s, "").trim().to_string() +} + +fn remove_season_and_episode(s: &str) -> String { + for re in SEASON_EPISODE_PATTERNS + .iter() + .chain(EPISODE_PATTERNS.iter()) + .chain(SEASON_ONLY_PATTERNS.iter()) + { + if re.is_match(s) { + return re.replace(s, "").trim().to_string(); + } + } + // removeRightmostNumber fallback. + SOLO_EP.replace(s, "").trim().to_string() +} + +/// Produce a clean show/movie title from a raw release name, faithful to +/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB +/// search — NOT a hand-rolled cleaner. +pub fn clean_title(raw: &str) -> String { + let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`) + s = remove_keywords(&s); // 172 release/quality keywords + v1..v5 + s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags + s = remove_beyond_hyphen(&s); // drop everything after the first " - " + s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets + s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear + s = SFV.replace(&s, "").into_owned(); // removeSFV + s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number + s = s.replace('.', " "); // cleanDot + let s = s.trim().trim_start_matches('-').trim_end_matches('-'); + s.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn se(title: &str) -> (Option, Option) { + let r = extract_season_episode(title); + (r.season, r.episode) + } + + #[test] + fn strict_sxxeyy() { + assert_eq!( + se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"), + (Some("1".into()), Some("2".into())) + ); + } + + #[test] + fn dotted_release_name() { + assert_eq!( + se("The.Wire.S03E07.720p.BluRay.x264"), + (Some("3".into()), Some("7".into())) + ); + } + + #[test] + fn clean_title_collapses_release_variants() { + let ct = |s: &str| clean_title(s).to_lowercase(); + // Season-word, underscore/range, dotted+EnN, and the trailing-num + // Nyaa shape all collapse to the bare show title (the cases that + // were failing TMDB enrichment). + assert_eq!( + ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), + "spy x family" + ); + assert_eq!( + ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"), + "spy x family" + ); + assert_eq!( + ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"), + "spy x family" + ); + assert_eq!( + ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"), + "spy x family" + ); + } + + #[test] + fn concise_nxnn() { + assert_eq!( + se("Some Show 1x05 720p x264"), + (Some("1".into()), Some("5".into())) + ); + } + + #[test] + fn bracketed_concise() { + assert_eq!( + se("Some Show [2x10] HDTV"), + (Some("2".into()), Some("10".into())) + ); + } + + #[test] + fn verbose_season_episode() { + assert_eq!( + se("My Show Season 2 Episode 13"), + (Some("2".into()), Some("13".into())) + ); + } + + #[test] + fn anime_trailing_number_is_season_less() { + // No season marker → episode via rightmost-number fallback, and NO + // season is fabricated (absolute numbering). SFV id stripped, brackets + // ignored. The UI buckets a season-less episode under "Episodes". + assert_eq!( + se("[SubsPlease] Some Anime - 117 [FC412C51]"), + (None, Some("117".into())) + ); + } + + #[test] + fn half_episode_special_is_preserved() { + assert_eq!( + se("Some Anime S01E13.5"), + (Some("1".into()), Some("13.5".into())) + ); + } + + #[test] + fn increment_is_season_times_10000_plus_episode() { + let r = extract_season_episode("Show S02E05"); + assert_eq!(r.increment.as_deref(), Some("20005")); + } + + #[test] + fn four_digit_year_is_not_misread_as_episode() { + // The `soloEp` fallback caps at 3 digits, so a 4-digit release + // year can't be picked up — no episode here. + let r = extract_season_episode("The Matrix 1999 1080p BluRay x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn short_number_in_movie_title_is_grabbed_by_fallback() { + // A bare 1-3 digit number in a movie title DOES trip the + // rightmost-number fallback (here: "13"). This is exactly why the + // torznab caller gates season/episode on TV-ness — a movie record + // must not inherit this bogus episode. + let r = extract_season_episode("Apollo 13 1080p BluRay x264"); + assert_eq!(r.episode.as_deref(), Some("13")); + } + + #[test] + fn keyword_strip_prevents_codec_false_positive() { + // "x264" must be removed so "264" isn't read as the episode. + let r = extract_season_episode("Plain Movie Title x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn no_numbers_yields_nothing() { + let r = extract_season_episode("Just A Plain Title"); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + } + + #[test] + fn season_only_pack_is_not_misread_as_episode() { + // Whole-season packs (no episode) must report the season with NO + // episode — not `S01E0N` from the rightmost-number fallback. This is + // the original bug: `Spy x Family Season 3 (S03)` was parsed as + // season 1, episode 3, dumping the S3 pack into the Season-1 bucket. + assert_eq!( + se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"), + (Some("3".into()), None) + ); + assert_eq!( + se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"), + (Some("2".into()), None) + ); + // Bare `S03` pack (no `Season` word) via the S-prefixed pattern. + assert_eq!( + se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"), + (Some("3".into()), None) + ); + assert_eq!( + se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"), + (Some("3".into()), None) + ); + // French season word. + assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None)); + } + + #[test] + fn explicit_sxxeyy_still_wins_over_season_only() { + // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected + // by the new season-only branch. + assert_eq!( + se("Spy x Family Season 3 S03E13 1080p"), + (Some("3".into()), Some("13".into())) + ); + } + + #[test] + fn episode_range_is_a_pack_not_sxxeyy() { + // The headline bug: `(01-10)` was mis-read as S1 E10 by the + // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no + // single `episode`. + let r = extract_season_episode("Frieren (01-28) [Batch] 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("28")); + + // Bare, dash-delimited, leading-zero range. + let r = extract_season_episode("Some Anime 01-12 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("12")); + + // Tilde range, no leading zero. + let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("26")); + assert_eq!(r.episode_end.as_deref(), Some("37")); + } + + #[test] + fn explicit_season_buckets_a_pack() { + // `S2 (01-10)` → pack in season 2, episodes 1–10. + let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("10")); + } + + #[test] + fn cjk_episode_range_is_a_pack() { + let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("29")); + assert_eq!(r.episode_end.as_deref(), Some("38")); + } + + #[test] + fn season_range_is_a_pack_with_no_single_season() { + let r = extract_season_episode("Frieren S01-S03 Complete 1080p"); + assert!(r.is_pack); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start, None); + } + + #[test] + fn whole_season_release_is_a_pack() { + // A season-only release (`Season 3`, `S03`, no episode) is a pack too — + // the original "season-level release shows as one media card" bug. + let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("3")); + assert_eq!(r.episode, None); + } + + #[test] + fn single_episode_is_not_a_pack() { + // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all + // single episodes, never packs. + for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] { + assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack"); + } + let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("7")); + + // The `Show - ` anime convention (spaced dash) is a + // single episode, NOT a `2..11` range — the tight-separator rule keeps + // this out of pack detection. + let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("11")); + + // A year range in a movie title is not an episode range. + assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack); + } + + #[test] + fn per_file_context_recovers_season_from_parent() { + // Opened pack: the inner file carries the episode, the pack title the + // season. `Frieren - 03.mkv` alone is season-less; with the S2 pack + // parent it resolves to S2E3 (the parent is the "season hint"). + let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv"); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("3")); + assert!(!r.is_pack); + + // The file's own explicit season wins over the parent. + let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv"); + assert_eq!(r.season.as_deref(), Some("1")); + assert_eq!(r.episode.as_deref(), Some("5")); + } + + #[test] + fn absolute_numbered_anime_is_season_less() { + // No season keyword → the rightmost-number fallback still finds the + // episode, but NO season is fabricated. Season is only ever asserted + // from an explicit token (title or parent), never inferred — so an + // absolute-numbered release lands in the UI's "Episodes" bucket. + assert_eq!( + se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"), + (None, Some("37".into())) + ); + } + + fn langs(s: &str) -> BTreeSet { + extract_languages(s) + } + + #[test] + fn language_from_release_tags() { + // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi). + assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre")); + assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger")); + assert!(langs("Show ENG SUB 720p").contains("eng")); + assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn")); + assert!(langs("Film TRUEFRENCH 1080p").contains("fre")); + } + + #[test] + fn multi_is_mul_and_collects_every_language() { + // `languages` is the union of all supported audio+sub languages. + // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is + // ALSO collected (the set, not a first-wins pick). + assert_eq!( + langs("Show MULTI VOSTFR 1080p"), + BTreeSet::from(["mul".to_string(), "fre".to_string()]) + ); + assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul")); + assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul")); + // A bilingual release collects both specific codes. + assert_eq!( + langs("Movie.2021.GERMAN.ENGLISH.1080p"), + BTreeSet::from(["ger".to_string(), "eng".to_string()]) + ); + } + + #[test] + fn no_language_tag_yields_empty_set() { + // A plain quality/codec release name carries no language tag — the + // caller maps the empty set to the `languages/und` sentinel. + assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty()); + // Short codes embedded in words/names must not false-positive: "ita" + // inside "digital", "por" inside "portal", etc. + assert!(langs("VFX Reel 2024 1080p").is_empty()); + assert!(langs("Digital Capital Portal 1080p").is_empty()); + } + + #[test] + fn lookaround_guards_reproduced_in_code() { + // WEB-DL must not read as `dl`→mul (German dual-language marker). + assert!(!langs("Movie 2021 WEB-DL x264").contains("mul")); + // "Shang-Chi" must not read as `chi` (Chinese)… + assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi")); + // …but a real CHI tag still does. + assert!(langs("Movie.2021.CHI.1080p").contains("chi")); + // "Tel Aviv" must not read as `tel` (Telugu). + assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel")); + // www. domain noise (incl. its TLD) must not contribute a language. + assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty()); + } + + #[test] + fn quality_tiers_and_normalization() { + let q = |s: &str| extract_quality(s); + assert_eq!( + q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(), + Some("1080p") + ); + assert_eq!( + q("The.Wire.S03E07.720p.BluRay.x264").as_deref(), + Some("720p") + ); + // 4k / interlaced collapse to the canonical progressive tier. + assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p")); + assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p")); + assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p")); + // WxH fallback, mapped by height; dot/underscore separators handled. + assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p")); + assert_eq!(q("Show 1280x720").as_deref(), Some("720p")); + // No resolution token → None (never guess). + assert_eq!(q("Just A Plain Title"), None); + } + + #[test] + fn codec_normalization() { + let c = |s: &str| extract_codec(s); + // HEVC encoder spellings collapse to the codec name. + assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc")); + assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc")); + assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc")); + // AVC encoder spellings collapse to h264. + assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264")); + assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264")); + // Others kept as-is. + assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1")); + assert_eq!(c("Old Release XviD").as_deref(), Some("xvid")); + // No codec token → None (never guess). + assert_eq!(c("Just A Plain Title 1080p"), None); + } +} diff --git a/src/hash.rs b/src/hash.rs new file mode 100644 index 0000000..2ed0cde --- /dev/null +++ b/src/hash.rs @@ -0,0 +1,1076 @@ +//! Content-addressed identifiers used by feeder plugins' `compute_outcomes` +//! to hash upstream-fetched bytes. +//! +//! Two function families live here: +//! +//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] — +//! the size-prefix-plus-middle-1MB-sample identification primitive shared +//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`. +//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the +//! middle slice directly so plugins that can only fetch a partial range +//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file +//! just to throw most of it away. **Rust-only by construction.** +//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full +//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec +//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches +//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +//! --hash=sha2-256 `. +//! +//! Cross-implementation parity for `compute_midhash256` with the TypeScript +//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression +//! silently breaks dedupe across the platform. + +use bytes::Bytes; +use sha2::{Digest, Sha256}; + +/// Sample window in bytes (1 MiB). Files this size or smaller are hashed +/// in full; larger files are hashed over the middle 1 MiB slice. +const SAMPLE_SIZE: usize = 1024 * 1024; + +/// Per-block IPFS chunk size (256 KiB), matches kubo's +/// `--chunker=size-262144` default. +pub const IPFS_CHUNK_SIZE: usize = 256 * 1024; + +/// dag-pb fanout — children per internal node. Matches kubo's balanced +/// builder default. +pub const IPFS_FANOUT: usize = 174; + +/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every +/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by +/// their own CID. Pass `.blocks` to the gateway's blockstore so peers +/// can fetch the file by CID via bitswap. +#[derive(Debug, Clone)] +pub struct IpfsBlocks { + /// Canonical "bafy…" / "bafk…" root cid string. Identical to what + /// [`compute_ipfs_cid`] returns for the same input. + pub root: String, + /// Every block — root included — keyed by its own cid. For a + /// single-leaf file, this is a one-entry vec `[(root, payload)]` + /// (raw codec, payload is the file bytes). For multi-leaf files, + /// it is `leaves ++ internal_nodes` in build order; the **last** + /// entry is the root. + pub blocks: Vec<(String, Bytes)>, +} + +/// Compute a midhash256 CID matching the TypeScript implementation in +/// `meta-hash/src/lib/file-id/FastHash.ts`: +/// +/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the +/// whole file if it fits in 1 MiB). +/// 2. SHA-256 it. +/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field +/// and the multihash hash-code field, length 32, then multibase-base32-lower +/// with the `b` prefix. +/// +/// **Endianness for the size prefix is big-endian** to match TS's +/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every +/// import — keep the test pinned. +pub fn compute_midhash256(bytes: &[u8]) -> String { + let size = bytes.len(); + let sample: &[u8] = if size <= SAMPLE_SIZE { + bytes + } else { + let start = (size - SAMPLE_SIZE) / 2; + &bytes[start..start + SAMPLE_SIZE] + }; + compute_midhash256_from_sample(size as u64, sample) +} + +/// Same midhash256 primitive as [`compute_midhash256`], but takes the +/// sample bytes directly rather than slicing them out of a full file. +/// For upstreams that can only deliver a middle range (BitTorrent BEP 9, +/// sliced HTTP), this avoids forcing the caller to buffer the whole file +/// in memory just to throw most of it away. +/// +/// `total_size` is the **full file size in bytes** — used as the +/// size prefix, NOT `middle_sample.len()`. +pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String { + let size_be = total_size.to_be_bytes(); + let mut hasher = Sha256::new(); + hasher.update(size_be); + hasher.update(middle_sample); + let digest: [u8; 32] = hasher.finalize().into(); + + // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20]. + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(38); + wire.push(0x01); + wire.extend_from_slice(&[0x80, 0x20]); + wire.extend_from_slice(&[0x80, 0x20]); + wire.push(0x20); + wire.extend_from_slice(&digest); + + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1. +/// +/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so +/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11), +/// len=20, digest=`. +pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String { + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(4 + 20); + wire.push(0x01); // CIDv1 version + wire.push(0x55); // codec: raw + wire.push(0x11); // multihash: sha1 + wire.push(0x14); // multihash length: 20 + wire.extend_from_slice(infohash_20); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a single file inside a **BitTorrent v1** torrent +/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`. +pub const BTIH_V1_FILE_CODEC: u64 = 0x1001; + +/// Custom multicodec for a single file inside a **BitTorrent v2** torrent +/// (`btih-v2-file`). +pub const BTIH_V2_FILE_CODEC: u64 = 0x1002; + +/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent, +/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info` +/// dict) plus the zero-based file index in `info.files`. Single-file +/// torrents use index `0`. +pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String { + bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index) +} + +/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent +/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's +/// canonical traversal index in `file tree`. +pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String { + bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index) +} + +/// Shared encoder for the two torrent-file CID families. `infohash` is 20 +/// bytes for v1 / 32 for v2; `codec` selects the family. +fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String { + // digest = infohash ‖ varint(file_index) + let mut digest = Vec::with_capacity(infohash.len() + 2); + digest.extend_from_slice(infohash); + write_pb_varint(file_index, &mut digest); + + // CIDv1: [version][codec varint][multihash code varint][len varint][digest] + let mut wire = Vec::with_capacity(1 + 4 + digest.len()); + wire.push(0x01); + write_pb_varint(codec, &mut wire); + write_pb_varint(codec, &mut wire); + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a self-describing **Newznab release** locator +/// (`nzb-release`). MetaMesh-private, adjacent to the torrent-file codecs in the +/// `0x10xx` range. Unlike a content hash, the cid *embeds* the indexer host + +/// release id in an identity multihash, so the credentialed meta-share peer +/// decodes `{host,id}` straight from the cid and grabs the `.nzb` via `t=get` +/// only at playback — no side-table, no `.nzb` download at search time. (The +/// `0x1004` `sha256(host‖id)` opaque hash and the `0x1003` `nzb-posting` +/// variant were both removed; only this self-describing form is emitted.) +pub const NZB_RELEASE_CODEC: u64 = 0x1005; + +/// Digest byte ceiling for an `nzb-release` cid, matching meta-share's +/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. +const NZB_RELEASE_MAX_DIGEST: usize = 64; + +/// Encode a self-describing `nzb-release` CID from a Newznab indexer's host + +/// bare release id (the hex `` id). The multihash is *identity* (`0x00`) +/// and its digest embeds the locator as `varint(host_len) ‖ host ‖ id_bytes`, +/// where `id_bytes` is the hex-decoded id. meta-share's `decode_nzb_release_cid` +/// is the exact inverse. Host-namespaced: the same release on two indexers +/// yields different cids. +/// +/// Returns `None` when the id isn't hex or the digest would overflow the +/// multihash budget (an oversized host) — the caller drops that row. +pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option { + let id_bytes = decode_hex_id(release_id)?; + let host_b = api_base.as_bytes(); + + // identity-multihash digest = varint(host_len) ‖ host ‖ id_bytes + let mut digest = Vec::with_capacity(2 + host_b.len() + id_bytes.len()); + write_pb_varint(host_b.len() as u64, &mut digest); + digest.extend_from_slice(host_b); + digest.extend_from_slice(&id_bytes); + if digest.len() > NZB_RELEASE_MAX_DIGEST { + return None; + } + + // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(NZB_RELEASE_CODEC, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + Some(format!("b{}", base32_lower_no_padding(&wire))) +} + +/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the +/// id is all-hex by construction; odd-length ids are left-padded with a `0` +/// nibble. Returns `None` on any non-hex byte (defensive). +fn decode_hex_id(id: &str) -> Option> { + let padded; + let s: &str = if id.len() % 2 == 1 { + padded = format!("0{id}"); + &padded + } else { + id + }; + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len() / 2); + for pair in bytes.chunks(2) { + let hi = (pair[0] as char).to_digit(16)?; + let lo = (pair[1] as char).to_digit(16)?; + out.push(((hi << 4) | lo) as u8); + } + Some(out) +} + +/// Custom multicodec for a **card locator** — the identity of a *work* (a +/// series, a film) as published by a metadata bridge, rather than of any +/// bytes. MetaMesh-private, the next free slot after `0x1006` url. (`0x1004` +/// nzb-release-hash stays **retired — do not reuse**. `0x1003` was retired too, +/// but has since been **revived** for the self-scanned Usenet posting identity — +/// see [`NZB_POSTING_CODEC`] and `METADATA_KEYS.md` §2.) +/// +/// This is the first CID family in the platform where **no bytes exist +/// anywhere, ever** — `0x1005`/`0x1006` are locators too, but both eventually +/// resolve to bytes (NNTP, HTTP), and `0x1000` midhash256 is a digest over +/// real file bytes. A card is pure identity: the record *is* the payload, and +/// it lives in meta-core. Consequences, all already handled: +/// +/// * Bitswap seeding is gated on `HashKind::Sha2_256` (gateway invariant 6), +/// so a card is never offered to the swarm — nothing to seed. +/// * meta-share's byte path 404s for it, which is correct. +/// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for +/// `canonical_cid` — which is what makes the future "one card, two CIDs" +/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe. +pub const CARD_LOCATOR_CODEC: u64 = 0x1007; + +/// Digest byte ceiling for a card locator, matching meta-share's +/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. +const CARD_LOCATOR_MAX_DIGEST: usize = 64; + +/// Encode a self-describing **card locator** CID from a metadata source and +/// that source's own id for the work (`("tmdb", "tv:95479")`, +/// `("mal", "52991")`). The multihash is *identity* (`0x00`) and its digest +/// embeds the locator as `varint(source_len) ‖ source ‖ id`, both UTF-8 — +/// exactly the shape [`compute_nzb_release_cid`] uses, minus the hex decode +/// (a card id is arbitrary text, not a hex release id). +/// +/// **The CID is a pure function of `(source, id)`**, which is the whole point: +/// any peer that knows a tmdb id can derive the card's address offline and +/// look it up locally, with no discovery round-trip. It also means the same +/// card discovered by two different peers converges on one meta-core record +/// for free, via the `cids/` reverse index. +/// +/// Source-namespaced: the same work on TMDB and on MyAnimeList yields +/// different cids. Merging those two into one record is a meta-core alias +/// operation driven by an explicit cross-source id mapping — never inferred +/// here (see `meta-gateway/docs/others/card-tier-search.md` §9). +/// +/// Returns `None` when `source` is empty or the digest would overflow the +/// multihash budget — the caller drops that card. +pub fn compute_card_cid(source: &str, id: &str) -> Option { + if source.is_empty() || id.is_empty() { + return None; + } + let source_b = source.as_bytes(); + let id_b = id.as_bytes(); + + // identity-multihash digest = varint(source_len) ‖ source ‖ id + let mut digest = Vec::with_capacity(2 + source_b.len() + id_b.len()); + write_pb_varint(source_b.len() as u64, &mut digest); + digest.extend_from_slice(source_b); + digest.extend_from_slice(id_b); + if digest.len() > CARD_LOCATOR_MAX_DIGEST { + return None; + } + + // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(CARD_LOCATOR_CODEC, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + Some(format!("b{}", base32_lower_no_padding(&wire))) +} + +// --------------------------------------------------------------------------- +// Delegated-playback locators — `yt-video` (0x1008) and `ext-play` (0x1009) +// --------------------------------------------------------------------------- +// +// These two are the limit case *beyond* `card` (0x1007). `card` addresses a +// work with no bytes anywhere; these address a **rendition whose bytes exist +// but are permanently someone else's**. An external player renders the media, +// nothing is transported, nothing is content-addressed, nothing is re-seedable. +// +// See `docs/cid-formats.md` §7 and +// `docs/study/listenbrainz-youtube-playback-tier-2026-08-21.md`. + +/// Custom multicodec for a **YouTube rendition** — delegated playback. +/// +/// Its own codec rather than a `url` (`0x1006`) for three reasons, in +/// descending order of how much trouble the alternative causes: +/// +/// * **`0x1006` means "fetch these bytes once and seed them".** Point a `url` +/// locator at a `watch` page and meta-share does exactly that: fetches the +/// HTML, chunks it, seeds it, and links the resulting content CID back onto +/// the record *as the file*. That is the MSR1 poisoning shape, except +/// permanent and replicated across peers. A resolved `googlevideo` URL is no +/// better — time-limited and IP-bound, so it is not a locator at all. +/// * **The id is canonical, a URL is not.** `youtube.com` / `music.youtube.com` +/// / `youtu.be` all address the same video, and query parameters are tracking +/// noise, so the same rendition would mint several different CIDs. Being a +/// pure function of `(kind, id)`, this codec makes two peers that resolve the +/// same track converge on one meta-core record for free, via the +/// `cids/` reverse index — the property that makes [`compute_card_cid`] +/// useful, for the same reason. +/// * **It is tiny** — 11 bytes of id plus a short kind prefix. +/// +/// ⚠ **An immutable id is not durable playback.** Takedowns, geo-blocks and +/// embed-disabled uploads all kill a reference whose CID stays valid forever — +/// the same rot class as [`NZB_RELEASE_CODEC`]. The mitigation is at the +/// *record* level (carry several ranked references per recording so a client +/// can fall through), never at the CID level. +pub const YT_VIDEO_CODEC: u64 = 0x1008; + +/// Custom multicodec for an **external page to open, never fetch**. +/// +/// Wire-identical to `url` (`0x1006`) apart from the codec, and that difference +/// is the entire contract: `0x1006` resolves cache-through and lands in the +/// blockstore, `0x1009` **must be refused by the byte path** and only ever +/// surfaces as an "Open on …↗" affordance. Use it for anything worth linking +/// but not worth a per-provider codec (Bandcamp, Jamendo, an arbitrary stream +/// page). +pub const EXT_PLAY_CODEC: u64 = 0x1009; + +/// Encode a [`YT_VIDEO_CODEC`] locator from a `kind` and a bare YouTube id. +/// +/// `kind` is `"video"` | `"playlist"` | `"channel"`; `id` is the bare +/// identifier (`4D7u5KF7SP8`, `OLAK5uy_…`, `UCRr1xG_2WIDs18a6cIiCxeA`) — **not** +/// a URL, and not a `watch?v=` fragment. The framing is byte-identical to +/// [`compute_card_cid`] on purpose, so each of the seven rank/decode +/// implementations extends a parser it already wrote rather than learning a +/// novel shape. +/// +/// Returns `None` on an empty field or a digest past the multihash budget — +/// the caller drops that reference. +pub fn compute_yt_video_cid(kind: &str, id: &str) -> Option { + kinded_locator_cid(YT_VIDEO_CODEC, kind, id) +} + +/// Encode an [`EXT_PLAY_CODEC`] locator from an `http(s)` URL. +/// +/// The scheme check is not cosmetic: it is what keeps a `javascript:` or +/// `data:` payload from ever reaching a client that will hand this string to +/// an anchor or a new tab. +pub fn compute_ext_play_cid(url: &str) -> Option { + if !(url.starts_with("http://") || url.starts_with("https://")) { + return None; + } + let digest = url.as_bytes(); + if digest.is_empty() || digest.len() > CARD_LOCATOR_MAX_DIGEST { + return None; + } + Some(identity_locator_cid(EXT_PLAY_CODEC, digest)) +} + +/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used +/// by [`compute_card_cid`] and [`compute_yt_video_cid`]. +fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option { + if kind.is_empty() || id.is_empty() { + return None; + } + let kind_b = kind.as_bytes(); + let id_b = id.as_bytes(); + + let mut digest = Vec::with_capacity(2 + kind_b.len() + id_b.len()); + write_pb_varint(kind_b.len() as u64, &mut digest); + digest.extend_from_slice(kind_b); + digest.extend_from_slice(id_b); + if digest.len() > CARD_LOCATOR_MAX_DIGEST { + return None; + } + Some(identity_locator_cid(codec, &digest)) +} + +/// CIDv1 with an *identity* multihash: +/// `[version=0x01][codec varint][mh=0x00][len varint][digest]`, base32lower. +fn identity_locator_cid(codec: u64, digest: &[u8]) -> String { + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(codec, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a **Usenet posting** identity (`nzb-posting`), +/// minted from the article Message-IDs of a release we scanned ourselves. +/// +/// Unlike [`NZB_RELEASE_CODEC`] (`0x1005`), this is **not** a locator: it +/// embeds no indexer host, so any peer with a plain NNTP provider can fetch the +/// articles — no `IndexerCred`, no `t=get` grab. That portability is the whole +/// point of running our own header scanner. +/// +/// It is also **not** fetchable by cid: the digest is over the Message-ID set, +/// not over any block's bytes, so bitswap can never satisfy a want for it (the +/// receiver derives a block's cid by hashing the block — see +/// `beetswap::incoming_stream`). The `.nzb` manifest travels separately, as an +/// ordinary sha2-256 IPFS cid in the record's `manifest` field. See +/// `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5. +pub const NZB_POSTING_CODEC: u64 = 0x1003; + +/// Mint an `nzb-posting` cid (`0x1003`) from a release's article Message-IDs. +/// +/// # The normalisation rule IS the dedup contract +/// +/// Mirrored verbatim in meta-share (`nzb/manifest.rs`), which re-derives this +/// digest from the fetched manifest and rejects a mismatch. **Any change here +/// must land on both sides together** — a drift silently makes every posting +/// unplayable. The rule: +/// +/// 1. Trim each id, and strip a surrounding `<…>` pair if present (NZB +/// `` bodies carry the bare form, but be defensive — NNTP `BODY` +/// re-adds the brackets). +/// 2. Drop empties. +/// 3. Deduplicate, then **sort** lexicographically by bytes. +/// 4. Join with `\n` and SHA-256 the result. +/// +/// Case is **preserved**: a Message-ID's local part is case-sensitive per RFC +/// 5322, so lowercasing could collide two genuinely distinct articles. +/// +/// Sorting is what makes the cid survive segment/file reordering between two +/// NZBs of the same posting — the reason we hash the *id set* and never the raw +/// `.nzb` bytes (the retired `0x1004` did the latter and broke dedup; see +/// meta-share `nzb/mod.rs`). +/// +/// **v1 scope:** every segment counts, par2 volumes included. Two NZBs for one +/// posting that disagree about whether to bundle par2 therefore mint different +/// cids. Acceptable while we are the only producer of these manifests; if a +/// second producer ever appears, the fix is to filter par2 subjects here *and* +/// in meta-share's mirror — do not do it on one side only. +pub fn compute_nzb_posting_cid>(message_ids: &[S]) -> String { + let mut ids: Vec<&str> = message_ids + .iter() + .map(|s| { + let t = s.as_ref().trim(); + t.strip_prefix('<') + .and_then(|r| r.strip_suffix('>')) + .unwrap_or(t) + }) + .filter(|s| !s.is_empty()) + .collect(); + ids.sort_unstable(); + ids.dedup(); + + let mut hasher = Sha256::new(); + for (i, id) in ids.iter().enumerate() { + if i > 0 { + hasher.update(b"\n"); + } + hasher.update(id.as_bytes()); + } + let digest: [u8; 32] = hasher.finalize().into(); + + // CIDv1: [version=0x01][codec varint][mh code=0x12 sha2-256][len=32][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(NZB_POSTING_CODEC, &mut wire); + wire.push(0x12); // multihash code: sha2-256 + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's +/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +/// --hash=sha2-256 `. +pub fn compute_ipfs_cid(bytes: &[u8]) -> String { + compute_ipfs_blocks(bytes).root +} + +/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every +/// block produced along the way (leaf chunks + internal dag-pb nodes) +/// so callers can populate a bitswap blockstore in one pass. +pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks { + // Leaves: chunk bytes into raw-codec blocks. Empty input still gets + // one (empty) leaf so the cid is well-defined. + let mut blocks: Vec<(String, Bytes)> = Vec::new(); + let leaves: Vec<(Vec, u64)> = if bytes.is_empty() { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b"")); + blocks.push((cid_string(&cid_wire), Bytes::new())); + vec![(cid_wire, 0)] + } else { + bytes + .chunks(IPFS_CHUNK_SIZE) + .map(|chunk| { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk)); + blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk))); + (cid_wire, chunk.len() as u64) + }) + .collect() + }; + + // Single-leaf file: return the leaf cid as the file cid. + if leaves.len() == 1 { + let root = cid_string(&leaves[0].0); + return IpfsBlocks { root, blocks }; + } + + // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes. + let mut level: Vec<(Vec, u64, u64)> = leaves + .into_iter() + .map(|(cid, size)| (cid, size, size)) + .collect(); + + while level.len() > 1 { + let mut next: Vec<(Vec, u64, u64)> = + Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT)); + for batch in level.chunks(IPFS_FANOUT) { + let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect(); + let filesize: u64 = blocksizes.iter().sum(); + let unixfs_data = encode_unixfs_file(filesize, &blocksizes); + let node_bytes = encode_dagpb_node(batch, &unixfs_data); + let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes)); + blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone()))); + let own_tsize: u64 = + batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64; + next.push((node_cid, filesize, own_tsize)); + } + level = next; + } + let root = cid_string(&level[0].0); + IpfsBlocks { root, blocks } +} + +fn cid_string(wire: &[u8]) -> String { + format!("b{}", base32_lower_no_padding(wire)) +} + +/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code +/// varint=0x12 sha2-256][len=0x20][digest...]`. +fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec { + debug_assert!( + codec < 0x80, + "codec varint must fit in one byte for this helper" + ); + let mut wire = Vec::with_capacity(36); + wire.push(0x01); // CIDv1 + wire.push(codec); + wire.push(0x12); // multihash code: sha2-256 + wire.push(0x20); // digest length: 32 + wire.extend_from_slice(digest); + wire +} + +fn sha2_256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// LEB128 unsigned varint encoding into `out`. +fn write_pb_varint(value: u64, out: &mut Vec) { + let mut v = value; + while v >= 0x80 { + out.push((v as u8) | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) { + write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint) + write_pb_varint(value, out); +} + +fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) { + write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited + write_pb_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +/// UnixFS protobuf payload for a File node: `Type=File, filesize, +/// blocksizes[]`. +fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec { + let mut out = Vec::new(); + write_pb_varint_field(1, 2, &mut out); // Type = File + write_pb_varint_field(3, filesize, &mut out); // filesize + for &bs in blocksizes { + write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated) + } + out +} + +/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order +/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second. +fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec { + let mut out = Vec::new(); + for (cid_wire, _filesize, tsize) in children { + let mut link_bytes = Vec::new(); + write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash + write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize + write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first + } + write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last + out +} + +/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the +/// multibase `b` prefix. +pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut out = String::with_capacity(input.len().div_ceil(5) * 8); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &b in input { + buffer = (buffer << 8) | b as u64; + bits += 8; + while bits >= 5 { + bits -= 5; + let idx = ((buffer >> bits) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + } + if bits > 0 { + let idx = ((buffer << (5 - bits)) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn midhash256_matches_ts_fixtures() { + assert_eq!( + compute_midhash256(b"hello world"), + "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4" + ); + assert_eq!( + compute_midhash256(b""), + "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y" + ); + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_midhash256(&mb_zeros), + "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4" + ); + } + + #[test] + fn midhash256_from_sample_matches_full_bytes() { + let small: &[u8] = b"hello world"; + assert_eq!( + compute_midhash256_from_sample(small.len() as u64, small), + compute_midhash256(small), + ); + + assert_eq!( + compute_midhash256_from_sample(0, b""), + compute_midhash256(b""), + ); + + let one_mb = vec![0u8; SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(one_mb.len() as u64, &one_mb), + compute_midhash256(&one_mb), + ); + + let two_mb = vec![0u8; 2 * SAMPLE_SIZE]; + let start = (two_mb.len() - SAMPLE_SIZE) / 2; + let middle = &two_mb[start..start + SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(two_mb.len() as u64, middle), + compute_midhash256(&two_mb), + ); + } + + #[test] + fn midhash256_from_sample_partial_sample_stable() { + let sample = vec![0u8; SAMPLE_SIZE]; + let total_5gb: u64 = 5 * 1024 * 1024 * 1024; + let total_10gb: u64 = 10 * 1024 * 1024 * 1024; + + let h1 = compute_midhash256_from_sample(total_5gb, &sample); + let h2 = compute_midhash256_from_sample(total_5gb, &sample); + assert_eq!(h1, h2, "same (size, sample) must hash identically"); + + let h3 = compute_midhash256_from_sample(total_10gb, &sample); + assert_ne!( + h1, h3, + "size prefix must contribute to cid — same sample, different size" + ); + } + + #[test] + fn bt_v1_file_cid_wire_layout() { + let infohash: [u8; 20] = [ + 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c, + 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06, + ]; + + let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want.extend_from_slice(&infohash); + want.push(0x04); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 4), + format!("b{}", base32_lower_no_padding(&want)) + ); + + let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want0.extend_from_slice(&infohash); + want0.push(0x00); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want0)) + ); + + let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16]; + want200.extend_from_slice(&infohash); + want200.extend_from_slice(&[0xC8, 0x01]); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 200), + format!("b{}", base32_lower_no_padding(&want200)) + ); + } + + #[test] + fn bt_v2_file_cid_wire_layout() { + let infohash: [u8; 32] = [0xAB; 32]; + let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21]; + want.extend_from_slice(&infohash); + want.push(0x00); + assert_eq!( + compute_bt_v2_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want)) + ); + } + + #[test] + fn bt_file_cid_properties() { + let ih1 = [0x11u8; 20]; + let ih2 = [0x22u8; 32]; + + assert_eq!( + compute_bt_v1_file_cid(&ih1, 3), + compute_bt_v1_file_cid(&ih1, 3) + ); + assert_ne!( + compute_bt_v1_file_cid(&ih1, 0), + compute_bt_v1_file_cid(&ih1, 1) + ); + assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b')); + assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b')); + assert_ne!( + compute_bt_v1_file_cid(&[0xAB; 20], 0), + compute_bt_v2_file_cid(&[0xAB; 32], 0) + ); + } + + #[test] + fn base32_known_vectors() { + assert_eq!(base32_lower_no_padding(b""), ""); + assert_eq!(base32_lower_no_padding(b"f"), "my"); + assert_eq!(base32_lower_no_padding(b"fo"), "mzxq"); + assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6"); + assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq"); + assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi"); + } + + #[test] + fn yt_video_cid_matches_golden_vector() { + // Pinned by `/cid-rank-vectors.json`'s `yt-video-locator` entry. The id + // is the real resolver output for Daft Punk / "Get Lucky" measured in + // the delegated-playback study §4.3. + assert_eq!( + compute_yt_video_cid("video", "4D7u5KF7SP8").unwrap(), + "bagecaaarav3gszdfn42ein3vgvfumn2tka4a" + ); + } + + #[test] + fn yt_video_cid_framing_matches_card_locator() { + // The two codecs share the `varint(len(kind)) || kind || id` framing on + // purpose, so every decoder extends a parser it already wrote. If this + // ever fails, the two shapes have drifted and §7.2 of docs/cid-formats.md + // is no longer true. + let yt = compute_yt_video_cid("tmdb", "tv:95479").unwrap(); + let card = compute_card_cid("tmdb", "tv:95479").unwrap(); + // Same digest, different codec => same length, differing only in the + // codec varint region. + assert_ne!(yt, card); + assert_eq!(yt.len(), card.len()); + } + + #[test] + fn yt_video_cid_is_a_pure_function_of_kind_and_id() { + // Convergence is the reason this is a codec and not a URL: two peers + // that resolve the same video must derive the same address offline. + assert_eq!( + compute_yt_video_cid("video", "4D7u5KF7SP8"), + compute_yt_video_cid("video", "4D7u5KF7SP8") + ); + // ...and the kind is part of the identity, not decoration. + assert_ne!( + compute_yt_video_cid("video", "OLAK5uy_kZ8Xq"), + compute_yt_video_cid("playlist", "OLAK5uy_kZ8Xq") + ); + } + + #[test] + fn yt_video_cid_rejects_empty_fields() { + assert!(compute_yt_video_cid("", "4D7u5KF7SP8").is_none()); + assert!(compute_yt_video_cid("video", "").is_none()); + } + + #[test] + fn ext_play_cid_matches_golden_vector() { + assert_eq!( + compute_ext_play_cid("https://example.bandcamp.com/album/x").unwrap(), + "bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa" + ); + } + + #[test] + fn ext_play_cid_rejects_non_http_schemes() { + // Load-bearing: a client hands this string to an anchor or a new tab, + // so a `javascript:` or `data:` payload must never mint a locator. + assert!(compute_ext_play_cid("javascript:alert(1)").is_none()); + assert!(compute_ext_play_cid("data:text/html,