inital commit

feeder-plugin/usenet-feeder/src/nntmux/db.rs

@@ -0,0 +1,281 @@
+//! Read-only access to the nntmux sidecar's release catalog (MariaDB).
+//!
+//! We are a **consumer** of nntmux's schema, never a writer: the scanner owns
+//! that database, and everything in it is re-derivable by re-scanning. Treat it
+//! as persistent cache, not as a source of truth (the Tribler precedent — a
+//! stateful sidecar the gateway still sees as a stateless HTTP feeder).
+//!
+//! Only a narrow, long-stable column set is selected (`guid`, `searchname`,
+//! `name`, `size`, `postdate`, `categories_id`, `totalpart`). nntmux carries
+//! ~80 columns on `releases`, most of them scanner bookkeeping; binding a wide
+//! row here would make us fragile to upstream schema churn for no gain.
+
+use anyhow::{Context, Result};
+use sqlx::mysql::{MySqlPool, MySqlPoolOptions};
+use sqlx::Row;
+
+/// One collated nntmux release — the metadata half of a Usenet posting.
+#[derive(Debug, Clone)]
+pub struct Release {
+    /// nntmux's public id, and the basename of the on-disk `<guid>.nzb.gz`.
+    /// This is our `record_id`.
+    pub guid: String,
+    /// The collator's cleaned, scene-style name
+    /// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`). Falls back to the
+    /// raw posted `name` when the collator could not improve on it.
+    pub search_name: String,
+    /// Total posted size in bytes (RAR + par2 included, so an over-estimate of
+    /// the final media file).
+    pub size: u64,
+    /// nntmux category id (`categories.id`). Coarse: 2000s = movies, 5000s = TV,
+    /// 3000s = audio, 7000s = books — the Newznab category tree.
+    pub category_id: i32,
+    /// Unix seconds of the original posting date, when nntmux recorded one.
+    pub post_date: Option<i64>,
+}
+
+/// Connection pool to nntmux's MariaDB. Small by design — this feeder issues a
+/// handful of short SELECTs per query, and the scanner needs the headroom.
+#[derive(Clone)]
+pub struct Db {
+    pool: MySqlPool,
+}
+
+impl Db {
+    /// Open a pool. Does **not** verify the schema — a feeder must come up and
+    /// serve `/health` even when its upstream is unreachable, so the gateway's
+    /// `depends_on` is satisfied and discovery does not soft-skip us forever.
+    pub async fn connect(url: &str) -> Result<Self> {
+        let pool = MySqlPoolOptions::new()
+            .max_connections(4)
+            .acquire_timeout(std::time::Duration::from_secs(10))
+            .connect(url)
+            .await
+            .with_context(|| "connect to nntmux mariadb")?;
+        Ok(Self { pool })
+    }
+
+    /// Free-text search over the collated release names.
+    ///
+    /// `LIKE %term%` on `searchname` — deliberately simple. nntmux can be built
+    /// with Manticore/Elasticsearch for real full-text search, but both are
+    /// optional in the minimum stack and we do not want to require them. If
+    /// search quality becomes the bottleneck, that is the upgrade path.
+    pub async fn search(&self, term: &str, limit: usize) -> Result<Vec<Release>> {
+        // ⚠ TOKENISE, don't match the phrase. Scene releases are dot-separated
+        // (`Breaking.Bad.S01E01.1080p...`) while users type spaces ("breaking
+        // bad"), so a single `LIKE %breaking bad%` matches essentially NOTHING
+        // — the separator never lines up. Split into words and AND them, which
+        // makes the separator irrelevant on both sides.
+        //
+        // Found by end-to-end search from meta-watch; the unit tests missed it
+        // because they only exercised the escaping, never a realistic
+        // scene-style name against a natural-language query.
+        let words = search_tokens(term);
+        if words.is_empty() {
+            return Ok(vec![]);
+        }
+        let mut sql = String::from(
+            "SELECT guid, searchname, name, size, categories_id, \
+                    UNIX_TIMESTAMP(postdate) AS postdate_unix \
+             FROM releases WHERE ",
+        );
+        sql.push_str(
+            &words
+                .iter()
+                .map(|_| "searchname LIKE ?")
+                .collect::<Vec<_>>()
+                .join(" AND "),
+        );
+        sql.push_str(" ORDER BY postdate DESC LIMIT ?");
+
+        let mut q = sqlx::query(&sql);
+        for w in &words {
+            q = q.bind(format!("%{}%", escape_like(w)));
+        }
+        let rows = q
+            .bind(limit as i64)
+            .fetch_all(&self.pool)
+            .await
+            .context("query releases by name")?;
+        Ok(rows.iter().map(row_to_release).collect())
+    }
+
+    /// Look one release up by guid — the `compute_outcomes` path, where the
+    /// record_id came back to us from the gateway.
+    pub async fn by_guid(&self, guid: &str) -> Result<Option<Release>> {
+        let row = sqlx::query(
+            "SELECT guid, searchname, name, size, categories_id, \
+                    UNIX_TIMESTAMP(postdate) AS postdate_unix \
+             FROM releases WHERE guid = ? LIMIT 1",
+        )
+        .bind(guid)
+        .fetch_optional(&self.pool)
+        .await
+        .context("query release by guid")?;
+        Ok(row.as_ref().map(row_to_release))
+    }
+
+    /// Cheap liveness probe for `health()` — confirms the catalog is reachable
+    /// *and* non-empty. An empty `releases` table is the visible symptom of both
+    /// nntmux silent-zero traps (an unset `PATH_TO_NZBS`, or groups left
+    /// `active=0`), so reporting it as degraded turns a silent failure into a
+    /// legible one.
+    pub async fn release_count(&self) -> Result<i64> {
+        let row = sqlx::query("SELECT COUNT(*) AS n FROM releases")
+            .fetch_one(&self.pool)
+            .await
+            .context("count releases")?;
+        Ok(row.try_get::<i64, _>("n").unwrap_or(0))
+    }
+}
+
+fn row_to_release(row: &sqlx::mysql::MySqlRow) -> Release {
+    let search_name: String = row
+        .try_get::<String, _>("searchname")
+        .ok()
+        .filter(|s| !s.is_empty())
+        .or_else(|| row.try_get::<String, _>("name").ok())
+        .unwrap_or_default();
+    Release {
+        guid: row.try_get::<String, _>("guid").unwrap_or_default(),
+        search_name,
+        // nntmux stores size as an unsigned bigint; it arrives as u64 or
+        // (older schemas) a decimal string.
+        size: row
+            .try_get::<u64, _>("size")
+            .ok()
+            .or_else(|| {
+                row.try_get::<String, _>("size")
+                    .ok()
+                    .and_then(|s| s.parse().ok())
+            })
+            .unwrap_or(0),
+        category_id: row.try_get::<i32, _>("categories_id").unwrap_or(0),
+        post_date: row.try_get::<i64, _>("postdate_unix").ok(),
+    }
+}
+
+/// Split a user query into match tokens.
+///
+/// Anything that is not alphanumeric is a separator — which folds the scene
+/// world's `.`/`_`/`-` and the human world's space onto the same footing, so
+/// "breaking bad", "Breaking.Bad" and "breaking-bad" all produce the same
+/// tokens and all match `Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE`.
+///
+/// Single characters are dropped: they match nearly every row and cost a full
+/// scan for no selectivity.
+fn search_tokens(term: &str) -> Vec<String> {
+    term.split(|c: char| !c.is_alphanumeric())
+        .filter(|w| w.chars().count() > 1)
+        .map(|w| w.to_string())
+        .collect()
+}
+
+/// Escape the `LIKE` wildcards so a user searching for `100%` does not match
+/// everything. Backslash is MySQL's default escape character.
+fn escape_like(s: &str) -> String {
+    s.replace('\\', "\\\\")
+        .replace('%', "\\%")
+        .replace('_', "\\_")
+}
+
+/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`.
+///
+/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s
+/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets
+/// `METADATA_KEYS.md` §1 defines.
+pub fn file_type_for_category(category_id: i32) -> &'static str {
+    match category_id / 1000 {
+        2 | 5 | 6 => "video",
+        3 => "audio",
+        7 => "document",
+        _ => "archive",
+    }
+}
+
+/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where
+/// the category tree is unambiguous — a wrong `contentKind` is worse than none,
+/// because meta-watch's curated rows gate on it.
+pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> {
+    match category_id / 1000 {
+        2 => Some("movie"),
+        5 => Some("tv"),
+        _ => None,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// ⚠ THE DOT-VS-SPACE REGRESSION. Scene releases are dot-separated and
+    /// users type spaces; matching the raw phrase finds nothing. Caught by an
+    /// end-to-end search from meta-watch that returned 0 results against a
+    /// catalog that plainly contained the show.
+    #[test]
+    fn a_spaced_query_tokenises_to_match_a_dotted_scene_name() {
+        let tokens = search_tokens("breaking bad");
+        assert_eq!(tokens, vec!["breaking", "bad"]);
+
+        // Every token must be a substring of the real scene name — that is
+        // exactly what the AND-ed LIKEs check.
+        let scene = "Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE".to_lowercase();
+        assert!(
+            tokens.iter().all(|t| scene.contains(&t.to_lowercase())),
+            "tokens {tokens:?} must all appear in {scene}"
+        );
+
+        // The naive whole-phrase form does NOT — this is the bug, pinned.
+        assert!(
+            !scene.contains("breaking bad"),
+            "if this ever passes, the dotted-name assumption changed"
+        );
+    }
+
+    /// Separators are interchangeable in both directions.
+    #[test]
+    fn scene_and_human_separators_produce_the_same_tokens() {
+        let expect = vec!["breaking".to_string(), "bad".to_string()];
+        assert_eq!(search_tokens("breaking bad"), expect);
+        assert_eq!(search_tokens("Breaking.Bad").iter().map(|s| s.to_lowercase()).collect::<Vec<_>>(), expect);
+        assert_eq!(search_tokens("breaking-bad"), expect);
+        assert_eq!(search_tokens("breaking_bad"), expect);
+        assert_eq!(search_tokens("  breaking   bad  "), expect);
+    }
+
+    /// Single characters are dropped (no selectivity, full scan) and an
+    /// all-noise query yields nothing rather than matching the whole catalog.
+    #[test]
+    fn noise_only_queries_yield_no_tokens() {
+        assert!(search_tokens("").is_empty());
+        assert!(search_tokens("   ").is_empty());
+        assert!(search_tokens("- . _").is_empty());
+        assert_eq!(search_tokens("a bc"), vec!["bc"]);
+    }
+
+    #[test]
+    fn like_wildcards_are_escaped() {
+        assert_eq!(escape_like("100%"), "100\\%");
+        assert_eq!(escape_like("a_b"), "a\\_b");
+        assert_eq!(escape_like("c:\\d"), "c:\\\\d");
+        assert_eq!(escape_like("naruto"), "naruto");
+    }
+
+    #[test]
+    fn newznab_categories_map_to_file_types() {
+        assert_eq!(file_type_for_category(2040), "video"); // movies HD
+        assert_eq!(file_type_for_category(5040), "video"); // tv HD
+        assert_eq!(file_type_for_category(3010), "audio");
+        assert_eq!(file_type_for_category(7020), "document"); // ebook
+        assert_eq!(file_type_for_category(4000), "archive"); // pc
+    }
+
+    #[test]
+    fn content_kind_only_where_unambiguous() {
+        assert_eq!(content_kind_for_category(2040), Some("movie"));
+        assert_eq!(content_kind_for_category(5040), Some("tv"));
+        assert_eq!(content_kind_for_category(3010), None);
+        assert_eq!(content_kind_for_category(7020), None);
+    }
+}

feeder-plugin/usenet-feeder/src/nntmux/nzb.rs

@@ -0,0 +1,187 @@
+//! Reading an nntmux release's manifest off disk and extracting its ordered
+//! article Message-IDs.
+//!
+//! # Why the file and not the database
+//!
+//! You **cannot** read a finished release's Message-IDs out of nntmux's
+//! database. `NzbService::writeNzbForReleaseId()` writes the release's NZB and
+//! then *unconditionally* purges collections → binaries → parts (all FKs are
+//! `ON DELETE CASCADE`, and there is no retention setting). Those tables are
+//! transient staging for the header scan; for any *collated* release the
+//! ordered Message-IDs survive **only** in the on-disk gzip NZB at
+//! `<nzb_root>/<split>/<guid>.nzb.gz`.
+//!
+//! This was confirmed on the live path during the spike (the offline
+//! `import-nzbs` path never stages collections/binaries/parts at all, so it
+//! cannot be used to check this). It is the single load-bearing fact behind the
+//! whole feeder: lose it and you are back to grabbing `.nzb` files from an
+//! external indexer, which is exactly what this design exists to avoid.
+
+use std::io::Read;
+use std::path::{Path, PathBuf};
+
+use anyhow::{Context, Result};
+use flate2::read::GzDecoder;
+
+/// A parsed nntmux manifest: the raw NZB XML plus the Message-IDs in posting
+/// order.
+#[derive(Debug, Clone)]
+pub struct Manifest {
+    /// The decompressed NZB XML. Handed to the gateway verbatim so it can seed
+    /// it as an ordinary sha2-256 cid — the consumer parses this exact document.
+    pub xml: Vec<u8>,
+    /// Every `<segment>` Message-ID across every `<file>`, in document order.
+    /// Bare form (no surrounding `<>`), matching how NZB stores them and how
+    /// meta-share's `NzbSegment.message_id` holds them.
+    pub message_ids: Vec<String>,
+}
+
+/// nntmux shards its NZB storage one level deep by the guid's **first
+/// character** (`storage/nzb/a/abcdef….nzb.gz`). Mirrored here rather than
+/// globbed: a scan of a large store is slow and this is on the compute path.
+pub fn nzb_path(nzb_root: &Path, guid: &str) -> Option<PathBuf> {
+    let first = guid.chars().next()?;
+    Some(nzb_root.join(first.to_string()).join(format!("{guid}.nzb.gz")))
+}
+
+/// Read + gunzip a release's `.nzb.gz` and pull out its Message-IDs.
+///
+/// Returns `Ok(None)` when the file is absent — a normal, non-fatal state: a
+/// release row can exist before `createNZBs()` has written its manifest, and
+/// the caller simply skips that record rather than failing the whole query.
+pub fn load(nzb_root: &Path, guid: &str) -> Result<Option<Manifest>> {
+    let Some(path) = nzb_path(nzb_root, guid) else {
+        return Ok(None);
+    };
+    if !path.exists() {
+        return Ok(None);
+    }
+    let raw = std::fs::read(&path).with_context(|| format!("read nzb {}", path.display()))?;
+    let mut xml = Vec::new();
+    GzDecoder::new(&raw[..])
+        .read_to_end(&mut xml)
+        .with_context(|| format!("gunzip nzb {}", path.display()))?;
+
+    let message_ids = extract_message_ids(&xml)
+        .with_context(|| format!("parse nzb {}", path.display()))?;
+    if message_ids.is_empty() {
+        anyhow::bail!("nzb {} has no segments", path.display());
+    }
+    Ok(Some(Manifest { xml, message_ids }))
+}
+
+/// Pull every `<segment>` body out of an NZB document, in order.
+///
+/// Deliberately a light scan rather than a full deserialize: we need exactly
+/// one thing from this document, the consumer (meta-share) does its own real
+/// parse of the same bytes, and a strict schema here would reject NZBs that
+/// meta-share would happily play. Surrounding `<>` are stripped if a producer
+/// included them — the mint normalises this too, but keeping the stored form
+/// canonical means the two never disagree about what was hashed.
+fn extract_message_ids(xml: &[u8]) -> Result<Vec<String>> {
+    use quick_xml::events::Event;
+    use quick_xml::Reader;
+
+    let text = std::str::from_utf8(xml).context("nzb is not utf-8")?;
+    let mut reader = Reader::from_str(text);
+    reader.config_mut().trim_text(true);
+
+    let mut ids = Vec::new();
+    let mut in_segment = false;
+    loop {
+        match reader.read_event() {
+            Ok(Event::Start(e)) if e.local_name().as_ref() == b"segment" => in_segment = true,
+            Ok(Event::End(e)) if e.local_name().as_ref() == b"segment" => in_segment = false,
+            Ok(Event::Text(t)) if in_segment => {
+                let raw = t.unescape().context("unescape segment text")?;
+                let id = raw
+                    .trim()
+                    .trim_start_matches('<')
+                    .trim_end_matches('>')
+                    .to_string();
+                if !id.is_empty() {
+                    ids.push(id);
+                }
+            }
+            Ok(Event::Eof) => break,
+            Err(e) => return Err(anyhow::anyhow!("nzb xml: {e}")),
+            _ => {}
+        }
+    }
+    Ok(ids)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const SAMPLE: &str = r#"<?xml version="1.0" encoding="iso-8859-1" ?>
+<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
+  <file poster="[email protected]" date="1" subject="Show.S01E01 [01/02] &quot;a.r00&quot; yEnc">
+    <groups><group>alt.binaries.teevee</group></groups>
+    <segments>
+      <segment bytes="100" number="1">[email protected]</segment>
+      <segment bytes="100" number="2">[email protected]</segment>
+    </segments>
+  </file>
+  <file poster="[email protected]" date="1" subject="Show.S01E01 [02/02] &quot;a.r01&quot; yEnc">
+    <segments>
+      <segment bytes="100" number="1">&lt;[email protected]&gt;</segment>
+    </segments>
+  </file>
+</nzb>"#;
+
+    #[test]
+    fn extracts_ids_in_document_order() {
+        let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse");
+        assert_eq!(
+            ids,
+            vec![
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+                "[email protected]".to_string(),
+            ]
+        );
+    }
+
+    /// Angle brackets are stripped so the stored form matches meta-share's
+    /// `NzbSegment.message_id` (bare) and the mint's normalisation.
+    #[test]
+    fn strips_angle_brackets() {
+        let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse");
+        assert!(ids.iter().all(|i| !i.contains('<') && !i.contains('>')));
+    }
+
+    #[test]
+    fn nzb_path_shards_on_first_guid_char() {
+        let p = nzb_path(Path::new("/s/nzb"), "abc123").expect("path");
+        assert_eq!(p, Path::new("/s/nzb/a/abc123.nzb.gz"));
+    }
+
+    /// A release row can exist before `createNZBs()` has written the manifest.
+    /// That is normal — the caller skips the record, it is not an error.
+    #[test]
+    fn missing_file_is_none_not_error() {
+        let dir = tempfile::tempdir().expect("tmp");
+        let got = load(dir.path(), "deadbeef").expect("no error");
+        assert!(got.is_none());
+    }
+
+    #[test]
+    fn round_trips_a_gzipped_nzb() {
+        use flate2::write::GzEncoder;
+        use flate2::Compression;
+        use std::io::Write;
+
+        let dir = tempfile::tempdir().expect("tmp");
+        std::fs::create_dir_all(dir.path().join("a")).expect("mkdir");
+        let mut enc = GzEncoder::new(Vec::new(), Compression::default());
+        enc.write_all(SAMPLE.as_bytes()).expect("gz write");
+        let gz = enc.finish().expect("gz finish");
+        std::fs::write(dir.path().join("a/abc.nzb.gz"), gz).expect("write");
+
+        let m = load(dir.path(), "abc").expect("load").expect("present");
+        assert_eq!(m.message_ids.len(), 3);
+        assert_eq!(m.xml, SAMPLE.as_bytes());
+    }
+}