Worph
@@ -0,0 +1,26 @@
|
||||
//! `usenet-feeder` library surface. The binary (`main.rs`) wraps the plugin in
|
||||
//! [`meta_feeder_sdk::serve_feeders`].
|
||||
//!
|
||||
//! This feeder owns the **metadata half** of Usenet: it reads an nntmux header-
|
||||
//! scan catalog we run ourselves and mints a portable `nzb-posting` (`0x1003`)
|
||||
//! cid from each release's article Message-IDs. Unlike `meta-feeder-indexer`'s
|
||||
//! `0x1005` locator, that cid embeds no indexer host, so any peer with a plain
|
||||
//! NNTP provider can redeem it — no indexer credential anywhere in the path.
|
||||
//!
|
||||
//! It ADDS a source alongside meta-feeder-indexer; it replaces nothing.
|
||||
//!
|
||||
//! Like every feeder it is meta-core-free and blockstore-free: it finds records
|
||||
//! and serves bytes, and the gateway core owns hashing-into-the-blockstore, the
|
||||
//! meta-core store-back, and the libp2p wire (gateway invariant 10).
|
||||
//!
|
||||
//! See `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md`.
|
||||
|
||||
/// Read-only access to the nntmux sidecar: its MariaDB release catalog and the
|
||||
/// on-disk gzip NZBs that are the *only* durable home of the Message-IDs.
|
||||
pub mod nntmux {
|
||||
pub mod db;
|
||||
pub mod nzb;
|
||||
}
|
||||
|
||||
/// The `usenet` upstream itself.
|
||||
pub mod usenet;
|
||||
@@ -0,0 +1,43 @@
|
||||
//! `usenet-feeder` — single-source Usenet feeder sidecar over an nntmux
|
||||
//! header-scan catalog.
|
||||
//!
|
||||
//! One plugin (`usenet`) served through [`meta_feeder_sdk::serve_feeders`].
|
||||
//!
|
||||
//! Operator config — the nntmux database URL and NZB store path — is read from
|
||||
//! the persisted `config.json` (dashboard-written), with the env vars below as a
|
||||
//! **first-boot seed only**: the file wins on the next restart, and there is no
|
||||
//! hot reload (gateway invariant 12).
|
||||
//!
|
||||
//! Env (seed): `NNTMUX_DB_URL`, `NNTMUX_NZB_PATH`.
|
||||
//! Env (infra): `META_FEEDER_HTTP_LISTEN` (default `0.0.0.0:8080`),
|
||||
//! `META_FEEDER_STATE_DIR` (default `/data/meta-feeder`), `RUST_LOG`.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use meta_feeder_sdk::plugin::FeederPlugin;
|
||||
use meta_feeder_sdk::serve_feeders;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use usenet_feeder::usenet::UsenetPlugin;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string())
|
||||
.parse()?;
|
||||
let state_dir =
|
||||
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
||||
|
||||
let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(UsenetPlugin::new())];
|
||||
|
||||
info!(target: "meta-feeder", loaded = plugins.len(), "usenet feeder starting");
|
||||
|
||||
serve_feeders(plugins, state_dir, listen).await
|
||||
}
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Read-only access to the nntmux sidecar's release catalog (MariaDB).
|
||||
//!
|
||||
//! We are a **consumer** of nntmux's schema, never a writer: the scanner owns
|
||||
//! that database, and everything in it is re-derivable by re-scanning. Treat it
|
||||
//! as persistent cache, not as a source of truth (the Tribler precedent — a
|
||||
//! stateful sidecar the gateway still sees as a stateless HTTP feeder).
|
||||
//!
|
||||
//! Only a narrow, long-stable column set is selected (`guid`, `searchname`,
|
||||
//! `name`, `size`, `postdate`, `categories_id`, `totalpart`). nntmux carries
|
||||
//! ~80 columns on `releases`, most of them scanner bookkeeping; binding a wide
|
||||
//! row here would make us fragile to upstream schema churn for no gain.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions};
|
||||
use sqlx::Row;
|
||||
|
||||
/// One collated nntmux release — the metadata half of a Usenet posting.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Release {
|
||||
/// nntmux's public id, and the basename of the on-disk `<guid>.nzb.gz`.
|
||||
/// This is our `record_id`.
|
||||
pub guid: String,
|
||||
/// The collator's cleaned, scene-style name
|
||||
/// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`). Falls back to the
|
||||
/// raw posted `name` when the collator could not improve on it.
|
||||
pub search_name: String,
|
||||
/// Total posted size in bytes (RAR + par2 included, so an over-estimate of
|
||||
/// the final media file).
|
||||
pub size: u64,
|
||||
/// nntmux category id (`categories.id`). Coarse: 2000s = movies, 5000s = TV,
|
||||
/// 3000s = audio, 7000s = books — the Newznab category tree.
|
||||
pub category_id: i32,
|
||||
/// Unix seconds of the original posting date, when nntmux recorded one.
|
||||
pub post_date: Option<i64>,
|
||||
}
|
||||
|
||||
/// Connection pool to nntmux's MariaDB. Small by design — this feeder issues a
|
||||
/// handful of short SELECTs per query, and the scanner needs the headroom.
|
||||
#[derive(Clone)]
|
||||
pub struct Db {
|
||||
pool: MySqlPool,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Open a pool. Does **not** verify the schema — a feeder must come up and
|
||||
/// serve `/health` even when its upstream is unreachable, so the gateway's
|
||||
/// `depends_on` is satisfied and discovery does not soft-skip us forever.
|
||||
pub async fn connect(url: &str) -> Result<Self> {
|
||||
let pool = MySqlPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.acquire_timeout(std::time::Duration::from_secs(10))
|
||||
.connect(url)
|
||||
.await
|
||||
.with_context(|| "connect to nntmux mariadb")?;
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
/// Free-text search over the collated release names.
|
||||
///
|
||||
/// `LIKE %term%` on `searchname` — deliberately simple. nntmux can be built
|
||||
/// with Manticore/Elasticsearch for real full-text search, but both are
|
||||
/// optional in the minimum stack and we do not want to require them. If
|
||||
/// search quality becomes the bottleneck, that is the upgrade path.
|
||||
pub async fn search(&self, term: &str, limit: usize) -> Result<Vec<Release>> {
|
||||
// ⚠ TOKENISE, don't match the phrase. Scene releases are dot-separated
|
||||
// (`Breaking.Bad.S01E01.1080p...`) while users type spaces ("breaking
|
||||
// bad"), so a single `LIKE %breaking bad%` matches essentially NOTHING
|
||||
// — the separator never lines up. Split into words and AND them, which
|
||||
// makes the separator irrelevant on both sides.
|
||||
//
|
||||
// Found by end-to-end search from meta-watch; the unit tests missed it
|
||||
// because they only exercised the escaping, never a realistic
|
||||
// scene-style name against a natural-language query.
|
||||
let words = search_tokens(term);
|
||||
if words.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut sql = String::from(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE ",
|
||||
);
|
||||
sql.push_str(
|
||||
&words
|
||||
.iter()
|
||||
.map(|_| "searchname LIKE ?")
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND "),
|
||||
);
|
||||
sql.push_str(" ORDER BY postdate DESC LIMIT ?");
|
||||
|
||||
let mut q = sqlx::query(&sql);
|
||||
for w in &words {
|
||||
q = q.bind(format!("%{}%", escape_like(w)));
|
||||
}
|
||||
let rows = q
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.context("query releases by name")?;
|
||||
Ok(rows.iter().map(row_to_release).collect())
|
||||
}
|
||||
|
||||
/// Look one release up by guid — the `compute_outcomes` path, where the
|
||||
/// record_id came back to us from the gateway.
|
||||
pub async fn by_guid(&self, guid: &str) -> Result<Option<Release>> {
|
||||
let row = sqlx::query(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE guid = ? LIMIT 1",
|
||||
)
|
||||
.bind(guid)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.context("query release by guid")?;
|
||||
Ok(row.as_ref().map(row_to_release))
|
||||
}
|
||||
|
||||
/// Cheap liveness probe for `health()` — confirms the catalog is reachable
|
||||
/// *and* non-empty. An empty `releases` table is the visible symptom of both
|
||||
/// nntmux silent-zero traps (an unset `PATH_TO_NZBS`, or groups left
|
||||
/// `active=0`), so reporting it as degraded turns a silent failure into a
|
||||
/// legible one.
|
||||
pub async fn release_count(&self) -> Result<i64> {
|
||||
let row = sqlx::query("SELECT COUNT(*) AS n FROM releases")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.context("count releases")?;
|
||||
Ok(row.try_get::<i64, _>("n").unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
fn row_to_release(row: &sqlx::mysql::MySqlRow) -> Release {
|
||||
let search_name: String = row
|
||||
.try_get::<String, _>("searchname")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.or_else(|| row.try_get::<String, _>("name").ok())
|
||||
.unwrap_or_default();
|
||||
Release {
|
||||
guid: row.try_get::<String, _>("guid").unwrap_or_default(),
|
||||
search_name,
|
||||
// nntmux stores size as an unsigned bigint; it arrives as u64 or
|
||||
// (older schemas) a decimal string.
|
||||
size: row
|
||||
.try_get::<u64, _>("size")
|
||||
.ok()
|
||||
.or_else(|| {
|
||||
row.try_get::<String, _>("size")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
})
|
||||
.unwrap_or(0),
|
||||
category_id: row.try_get::<i32, _>("categories_id").unwrap_or(0),
|
||||
post_date: row.try_get::<i64, _>("postdate_unix").ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a user query into match tokens.
|
||||
///
|
||||
/// Anything that is not alphanumeric is a separator — which folds the scene
|
||||
/// world's `.`/`_`/`-` and the human world's space onto the same footing, so
|
||||
/// "breaking bad", "Breaking.Bad" and "breaking-bad" all produce the same
|
||||
/// tokens and all match `Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE`.
|
||||
///
|
||||
/// Single characters are dropped: they match nearly every row and cost a full
|
||||
/// scan for no selectivity.
|
||||
fn search_tokens(term: &str) -> Vec<String> {
|
||||
term.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| w.chars().count() > 1)
|
||||
.map(|w| w.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Escape the `LIKE` wildcards so a user searching for `100%` does not match
|
||||
/// everything. Backslash is MySQL's default escape character.
|
||||
fn escape_like(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_")
|
||||
}
|
||||
|
||||
/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`.
|
||||
///
|
||||
/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s
|
||||
/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets
|
||||
/// `METADATA_KEYS.md` §1 defines.
|
||||
pub fn file_type_for_category(category_id: i32) -> &'static str {
|
||||
match category_id / 1000 {
|
||||
2 | 5 | 6 => "video",
|
||||
3 => "audio",
|
||||
7 => "document",
|
||||
_ => "archive",
|
||||
}
|
||||
}
|
||||
|
||||
/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where
|
||||
/// the category tree is unambiguous — a wrong `contentKind` is worse than none,
|
||||
/// because meta-watch's curated rows gate on it.
|
||||
pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> {
|
||||
match category_id / 1000 {
|
||||
2 => Some("movie"),
|
||||
5 => Some("tv"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// ⚠ THE DOT-VS-SPACE REGRESSION. Scene releases are dot-separated and
|
||||
/// users type spaces; matching the raw phrase finds nothing. Caught by an
|
||||
/// end-to-end search from meta-watch that returned 0 results against a
|
||||
/// catalog that plainly contained the show.
|
||||
#[test]
|
||||
fn a_spaced_query_tokenises_to_match_a_dotted_scene_name() {
|
||||
let tokens = search_tokens("breaking bad");
|
||||
assert_eq!(tokens, vec!["breaking", "bad"]);
|
||||
|
||||
// Every token must be a substring of the real scene name — that is
|
||||
// exactly what the AND-ed LIKEs check.
|
||||
let scene = "Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE".to_lowercase();
|
||||
assert!(
|
||||
tokens.iter().all(|t| scene.contains(&t.to_lowercase())),
|
||||
"tokens {tokens:?} must all appear in {scene}"
|
||||
);
|
||||
|
||||
// The naive whole-phrase form does NOT — this is the bug, pinned.
|
||||
assert!(
|
||||
!scene.contains("breaking bad"),
|
||||
"if this ever passes, the dotted-name assumption changed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Separators are interchangeable in both directions.
|
||||
#[test]
|
||||
fn scene_and_human_separators_produce_the_same_tokens() {
|
||||
let expect = vec!["breaking".to_string(), "bad".to_string()];
|
||||
assert_eq!(search_tokens("breaking bad"), expect);
|
||||
assert_eq!(search_tokens("Breaking.Bad").iter().map(|s| s.to_lowercase()).collect::<Vec<_>>(), expect);
|
||||
assert_eq!(search_tokens("breaking-bad"), expect);
|
||||
assert_eq!(search_tokens("breaking_bad"), expect);
|
||||
assert_eq!(search_tokens(" breaking bad "), expect);
|
||||
}
|
||||
|
||||
/// Single characters are dropped (no selectivity, full scan) and an
|
||||
/// all-noise query yields nothing rather than matching the whole catalog.
|
||||
#[test]
|
||||
fn noise_only_queries_yield_no_tokens() {
|
||||
assert!(search_tokens("").is_empty());
|
||||
assert!(search_tokens(" ").is_empty());
|
||||
assert!(search_tokens("- . _").is_empty());
|
||||
assert_eq!(search_tokens("a bc"), vec!["bc"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn like_wildcards_are_escaped() {
|
||||
assert_eq!(escape_like("100%"), "100\\%");
|
||||
assert_eq!(escape_like("a_b"), "a\\_b");
|
||||
assert_eq!(escape_like("c:\\d"), "c:\\\\d");
|
||||
assert_eq!(escape_like("naruto"), "naruto");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newznab_categories_map_to_file_types() {
|
||||
assert_eq!(file_type_for_category(2040), "video"); // movies HD
|
||||
assert_eq!(file_type_for_category(5040), "video"); // tv HD
|
||||
assert_eq!(file_type_for_category(3010), "audio");
|
||||
assert_eq!(file_type_for_category(7020), "document"); // ebook
|
||||
assert_eq!(file_type_for_category(4000), "archive"); // pc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_kind_only_where_unambiguous() {
|
||||
assert_eq!(content_kind_for_category(2040), Some("movie"));
|
||||
assert_eq!(content_kind_for_category(5040), Some("tv"));
|
||||
assert_eq!(content_kind_for_category(3010), None);
|
||||
assert_eq!(content_kind_for_category(7020), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
//! Reading an nntmux release's manifest off disk and extracting its ordered
|
||||
//! article Message-IDs.
|
||||
//!
|
||||
//! # Why the file and not the database
|
||||
//!
|
||||
//! You **cannot** read a finished release's Message-IDs out of nntmux's
|
||||
//! database. `NzbService::writeNzbForReleaseId()` writes the release's NZB and
|
||||
//! then *unconditionally* purges collections → binaries → parts (all FKs are
|
||||
//! `ON DELETE CASCADE`, and there is no retention setting). Those tables are
|
||||
//! transient staging for the header scan; for any *collated* release the
|
||||
//! ordered Message-IDs survive **only** in the on-disk gzip NZB at
|
||||
//! `<nzb_root>/<split>/<guid>.nzb.gz`.
|
||||
//!
|
||||
//! This was confirmed on the live path during the spike (the offline
|
||||
//! `import-nzbs` path never stages collections/binaries/parts at all, so it
|
||||
//! cannot be used to check this). It is the single load-bearing fact behind the
|
||||
//! whole feeder: lose it and you are back to grabbing `.nzb` files from an
|
||||
//! external indexer, which is exactly what this design exists to avoid.
|
||||
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use flate2::read::GzDecoder;
|
||||
|
||||
/// A parsed nntmux manifest: the raw NZB XML plus the Message-IDs in posting
|
||||
/// order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Manifest {
|
||||
/// The decompressed NZB XML. Handed to the gateway verbatim so it can seed
|
||||
/// it as an ordinary sha2-256 cid — the consumer parses this exact document.
|
||||
pub xml: Vec<u8>,
|
||||
/// Every `<segment>` Message-ID across every `<file>`, in document order.
|
||||
/// Bare form (no surrounding `<>`), matching how NZB stores them and how
|
||||
/// meta-share's `NzbSegment.message_id` holds them.
|
||||
pub message_ids: Vec<String>,
|
||||
}
|
||||
|
||||
/// nntmux shards its NZB storage one level deep by the guid's **first
|
||||
/// character** (`storage/nzb/a/abcdef….nzb.gz`). Mirrored here rather than
|
||||
/// globbed: a scan of a large store is slow and this is on the compute path.
|
||||
pub fn nzb_path(nzb_root: &Path, guid: &str) -> Option<PathBuf> {
|
||||
let first = guid.chars().next()?;
|
||||
Some(nzb_root.join(first.to_string()).join(format!("{guid}.nzb.gz")))
|
||||
}
|
||||
|
||||
/// Read + gunzip a release's `.nzb.gz` and pull out its Message-IDs.
|
||||
///
|
||||
/// Returns `Ok(None)` when the file is absent — a normal, non-fatal state: a
|
||||
/// release row can exist before `createNZBs()` has written its manifest, and
|
||||
/// the caller simply skips that record rather than failing the whole query.
|
||||
pub fn load(nzb_root: &Path, guid: &str) -> Result<Option<Manifest>> {
|
||||
let Some(path) = nzb_path(nzb_root, guid) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw = std::fs::read(&path).with_context(|| format!("read nzb {}", path.display()))?;
|
||||
let mut xml = Vec::new();
|
||||
GzDecoder::new(&raw[..])
|
||||
.read_to_end(&mut xml)
|
||||
.with_context(|| format!("gunzip nzb {}", path.display()))?;
|
||||
|
||||
let message_ids = extract_message_ids(&xml)
|
||||
.with_context(|| format!("parse nzb {}", path.display()))?;
|
||||
if message_ids.is_empty() {
|
||||
anyhow::bail!("nzb {} has no segments", path.display());
|
||||
}
|
||||
Ok(Some(Manifest { xml, message_ids }))
|
||||
}
|
||||
|
||||
/// Pull every `<segment>` body out of an NZB document, in order.
|
||||
///
|
||||
/// Deliberately a light scan rather than a full deserialize: we need exactly
|
||||
/// one thing from this document, the consumer (meta-share) does its own real
|
||||
/// parse of the same bytes, and a strict schema here would reject NZBs that
|
||||
/// meta-share would happily play. Surrounding `<>` are stripped if a producer
|
||||
/// included them — the mint normalises this too, but keeping the stored form
|
||||
/// canonical means the two never disagree about what was hashed.
|
||||
fn extract_message_ids(xml: &[u8]) -> Result<Vec<String>> {
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader;
|
||||
|
||||
let text = std::str::from_utf8(xml).context("nzb is not utf-8")?;
|
||||
let mut reader = Reader::from_str(text);
|
||||
reader.config_mut().trim_text(true);
|
||||
|
||||
let mut ids = Vec::new();
|
||||
let mut in_segment = false;
|
||||
loop {
|
||||
match reader.read_event() {
|
||||
Ok(Event::Start(e)) if e.local_name().as_ref() == b"segment" => in_segment = true,
|
||||
Ok(Event::End(e)) if e.local_name().as_ref() == b"segment" => in_segment = false,
|
||||
Ok(Event::Text(t)) if in_segment => {
|
||||
let raw = t.unescape().context("unescape segment text")?;
|
||||
let id = raw
|
||||
.trim()
|
||||
.trim_start_matches('<')
|
||||
.trim_end_matches('>')
|
||||
.to_string();
|
||||
if !id.is_empty() {
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(anyhow::anyhow!("nzb xml: {e}")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(ids)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SAMPLE: &str = r#"<?xml version="1.0" encoding="iso-8859-1" ?>
|
||||
<nzb xmlns="http://www.newzbin.com/DTD/2003/nzb">
|
||||
<file poster="[email protected]" date="1" subject="Show.S01E01 [01/02] "a.r00" yEnc">
|
||||
<groups><group>alt.binaries.teevee</group></groups>
|
||||
<segments>
|
||||
<segment bytes="100" number="1">[email protected]</segment>
|
||||
<segment bytes="100" number="2">[email protected]</segment>
|
||||
</segments>
|
||||
</file>
|
||||
<file poster="[email protected]" date="1" subject="Show.S01E01 [02/02] "a.r01" yEnc">
|
||||
<segments>
|
||||
<segment bytes="100" number="1"><[email protected]></segment>
|
||||
</segments>
|
||||
</file>
|
||||
</nzb>"#;
|
||||
|
||||
#[test]
|
||||
fn extracts_ids_in_document_order() {
|
||||
let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse");
|
||||
assert_eq!(
|
||||
ids,
|
||||
vec![
|
||||
"[email protected]".to_string(),
|
||||
"[email protected]".to_string(),
|
||||
"[email protected]".to_string(),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// Angle brackets are stripped so the stored form matches meta-share's
|
||||
/// `NzbSegment.message_id` (bare) and the mint's normalisation.
|
||||
#[test]
|
||||
fn strips_angle_brackets() {
|
||||
let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse");
|
||||
assert!(ids.iter().all(|i| !i.contains('<') && !i.contains('>')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nzb_path_shards_on_first_guid_char() {
|
||||
let p = nzb_path(Path::new("/s/nzb"), "abc123").expect("path");
|
||||
assert_eq!(p, Path::new("/s/nzb/a/abc123.nzb.gz"));
|
||||
}
|
||||
|
||||
/// A release row can exist before `createNZBs()` has written the manifest.
|
||||
/// That is normal — the caller skips the record, it is not an error.
|
||||
#[test]
|
||||
fn missing_file_is_none_not_error() {
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
let got = load(dir.path(), "deadbeef").expect("no error");
|
||||
assert!(got.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_a_gzipped_nzb() {
|
||||
use flate2::write::GzEncoder;
|
||||
use flate2::Compression;
|
||||
use std::io::Write;
|
||||
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
std::fs::create_dir_all(dir.path().join("a")).expect("mkdir");
|
||||
let mut enc = GzEncoder::new(Vec::new(), Compression::default());
|
||||
enc.write_all(SAMPLE.as_bytes()).expect("gz write");
|
||||
let gz = enc.finish().expect("gz finish");
|
||||
std::fs::write(dir.path().join("a/abc.nzb.gz"), gz).expect("write");
|
||||
|
||||
let m = load(dir.path(), "abc").expect("load").expect("present");
|
||||
assert_eq!(m.message_ids.len(), 3);
|
||||
assert_eq!(m.xml, SAMPLE.as_bytes());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
//! The `usenet` upstream — a `FeederPlugin` over an nntmux header-scan catalog.
|
||||
//!
|
||||
//! # What makes this different from `meta-feeder-indexer`
|
||||
//!
|
||||
//! indexer-feeder proxies an **external** Newznab indexer and mints an
|
||||
//! `nzb-release` (`0x1005`) cid that *embeds that indexer's host*. Only a peer
|
||||
//! holding that indexer's credential can ever redeem it.
|
||||
//!
|
||||
//! This feeder scans Usenet itself, so it already holds the article Message-IDs
|
||||
//! at collation time — no grab, no scrape. That lets it mint a **portable**
|
||||
//! `nzb-posting` (`0x1003`) cid: a digest over the Message-ID set, embedding no
|
||||
//! host, redeemable by any peer with a plain NNTP provider.
|
||||
//!
|
||||
//! The two coexist. This ADDS a Usenet metadata source; it removes nothing.
|
||||
//!
|
||||
//! # The manifest hand-off (the one piece of new machinery)
|
||||
//!
|
||||
//! A `0x1003` cid is a *hash*, not a locator: it is not reversible to the
|
||||
//! Message-IDs, and it can never be bitswap-fetched under itself (bitswap
|
||||
//! derives a block's cid by hashing the block). So the `.nzb` must travel
|
||||
//! separately. We emit `manifest_url` — a **relative** URL onto our own
|
||||
//! `/blob/...` route — and the gateway core fetches it, seeds it as an ordinary
|
||||
//! sha2-256 IPFS cid, publishes it as a first-class `fileType=nzb` meta-core
|
||||
//! record, and rewrites the field to that cid. Identical to how posters are
|
||||
//! handled. See the study's §5.3.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use meta_feeder_sdk::config::{ConfigField as F, ConfigSchema};
|
||||
use meta_feeder_sdk::hash::compute_nzb_posting_cid;
|
||||
use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome};
|
||||
use meta_feeder_sdk::query::GatewayQuery;
|
||||
use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::nntmux::db::{content_kind_for_category, file_type_for_category, Db, Release};
|
||||
use crate::nntmux::nzb;
|
||||
|
||||
pub const UPSTREAM_ID: &str = "usenet";
|
||||
|
||||
/// Operator-supplied settings, persisted to `<cache_dir>/config.json` by the
|
||||
/// SDK's config surface. Env is a **seed only** — the file wins on the next
|
||||
/// restart (invariant 12; there is no hot reload).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Settings {
|
||||
/// `mysql://user:pass@host:3306/nntmux`
|
||||
pub db_url: String,
|
||||
/// Root of nntmux's NZB store, shared with the sidecar as a volume. The
|
||||
/// directory nntmux's `PATH_TO_NZBS` points at.
|
||||
pub nzb_root: String,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
db_url: std::env::var("NNTMUX_DB_URL").unwrap_or_default(),
|
||||
nzb_root: std::env::var("NNTMUX_NZB_PATH")
|
||||
.unwrap_or_else(|_| "/nntmux-nzb".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_file(&mut self, cache_dir: &Path) {
|
||||
let path = cache_dir.join("config.json");
|
||||
let Ok(raw) = std::fs::read_to_string(&path) else {
|
||||
return;
|
||||
};
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&raw) else {
|
||||
warn!(target: "meta-feeder", path = %path.display(), "config.json is not valid json; using env seed");
|
||||
return;
|
||||
};
|
||||
if let Some(s) = v.get("db_url").and_then(|x| x.as_str()) {
|
||||
if !s.is_empty() {
|
||||
self.db_url = s.to_string();
|
||||
}
|
||||
}
|
||||
if let Some(s) = v.get("nzb_root").and_then(|x| x.as_str()) {
|
||||
if !s.is_empty() {
|
||||
self.nzb_root = s.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct UsenetPlugin {
|
||||
settings: Settings,
|
||||
/// `None` until a successful connect. A feeder with unreachable config
|
||||
/// **must still serve `/health`** so the gateway's `depends_on` is satisfied
|
||||
/// — it soft-skips its upstream instead of failing to boot (invariant 10).
|
||||
db: Arc<RwLock<Option<Db>>>,
|
||||
nzb_root: PathBuf,
|
||||
}
|
||||
|
||||
impl Default for UsenetPlugin {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl UsenetPlugin {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
settings: Settings::default(),
|
||||
db: Arc::new(RwLock::new(None)),
|
||||
nzb_root: PathBuf::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn db(&self) -> Result<Db, GatewayError> {
|
||||
self.db
|
||||
.read()
|
||||
.await
|
||||
.clone()
|
||||
.ok_or_else(|| GatewayError::Permanent("nntmux database not configured".into()))
|
||||
}
|
||||
|
||||
/// Mint a release's `nzb-posting` cid by reading its manifest off disk.
|
||||
///
|
||||
/// `None` when the `.nzb.gz` is absent (a release row can exist before
|
||||
/// `createNZBs()` has written it) or unreadable — the caller then emits a
|
||||
/// record without a cid, which is honest: we genuinely cannot address that
|
||||
/// posting yet.
|
||||
fn posting_cid(&self, guid: &str) -> Option<String> {
|
||||
match nzb::load(&self.nzb_root, guid) {
|
||||
Ok(Some(m)) => Some(compute_nzb_posting_cid(&m.message_ids)),
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
warn!(target: "meta-feeder", %guid, error = %e, "usenet: manifest unreadable; record will carry no cid");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the discovery record for one release. Metadata only — the bytes
|
||||
/// live on Usenet and are fetched by meta-share at playback, never by us.
|
||||
///
|
||||
/// ⚠ **`cid` must be `Some` on the SEARCH path, not just on compute.** A
|
||||
/// record whose fields carry no `cids/<cid>` member reaches the client as a
|
||||
/// bare `gateway:<upstream>:<record_id>` reference, which meta-share
|
||||
/// refuses to parse (the `<algo>:<cid>` token form was removed) — so the
|
||||
/// title renders but is unplayable. `nzb-release` (`0x1005`) avoids this
|
||||
/// for free because its cid is derivable from `{host, id}` alone; ours is a
|
||||
/// digest over the Message-ID set, so it has to be minted from the manifest
|
||||
/// — which is a **local file read**, no network and no indexer grab, and
|
||||
/// therefore fine to do per search hit. Caught by an end-to-end play from
|
||||
/// meta-watch that landed on "Unavailable".
|
||||
fn record_for(&self, r: &Release, cid: Option<&str>) -> DiscoveryRecord {
|
||||
let mut fields: BTreeMap<String, String> = BTreeMap::new();
|
||||
// The bare-cid key-set (`cids/<cid>` = "true") — the ONLY shape
|
||||
// meta-core indexes and the one meta-search elects the canonical cid
|
||||
// from (METADATA_KEYS §2). A flat `cid` field would be inert.
|
||||
if let Some(c) = cid {
|
||||
fields.insert(format!("cids/{c}"), "true".to_string());
|
||||
// ⚠ EMITTED ON THE SEARCH PATH, NOT JUST ON COMPUTE. The gateway
|
||||
// core rewrites this to a content-addressed `manifest` cid (its
|
||||
// SEEDABLE_FIELDS table) and persists the record — and it does that
|
||||
// for *search* hits, because the search path is the one that
|
||||
// actually runs: `GatewayOp::Hash` (compute) is not sent by any
|
||||
// client in the fleet today, so anything emitted only from
|
||||
// `compute_outcomes` is never seen. Without this on search, the
|
||||
// stored record has no `manifest`, and meta-share cannot resolve
|
||||
// the posting at playback ("Unavailable" in meta-watch).
|
||||
//
|
||||
// Relative on purpose: we do not know our own externally-reachable
|
||||
// address; the core resolves it against the URL it discovered us
|
||||
// on (`resolve_seed_url`).
|
||||
fields.insert(
|
||||
"manifest_url".to_string(),
|
||||
format!("/blob/{UPSTREAM_ID}/{}.nzb", r.guid),
|
||||
);
|
||||
}
|
||||
// Canonical per-upstream id field (`<upstream_id>id`) — required of
|
||||
// every record by the SDK's field-naming convention.
|
||||
fields.insert(format!("{UPSTREAM_ID}id"), r.guid.clone());
|
||||
fields.insert("title".to_string(), r.search_name.clone());
|
||||
fields.insert("fileName".to_string(), r.search_name.clone());
|
||||
if r.size > 0 {
|
||||
fields.insert("sizeByte".to_string(), r.size.to_string());
|
||||
}
|
||||
fields.insert(
|
||||
"fileType".to_string(),
|
||||
file_type_for_category(r.category_id).to_string(),
|
||||
);
|
||||
if let Some(kind) = content_kind_for_category(r.category_id) {
|
||||
fields.insert("contentKind".to_string(), kind.to_string());
|
||||
}
|
||||
if let Some(ts) = r.post_date {
|
||||
fields.insert("publishedAt".to_string(), ts.to_string());
|
||||
}
|
||||
// Provenance: this catalog is ours, not an external indexer's. Kept
|
||||
// distinct from indexer-feeder's `indexer` field on purpose — an
|
||||
// operator reading a record should be able to tell at a glance whether
|
||||
// its metadata came from a third party or from our own scan.
|
||||
fields.insert("source/gateway:usenet-scan".to_string(), "true".to_string());
|
||||
DiscoveryRecord {
|
||||
upstream_id: UPSTREAM_ID.to_string(),
|
||||
record_id: r.guid.clone(),
|
||||
fields,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FeederPlugin for UsenetPlugin {
|
||||
fn upstream_id(&self) -> &'static str {
|
||||
UPSTREAM_ID
|
||||
}
|
||||
|
||||
fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
|
||||
let mut settings = Settings::from_env();
|
||||
settings.merge_file(cache_dir);
|
||||
self.nzb_root = PathBuf::from(&settings.nzb_root);
|
||||
|
||||
if settings.db_url.is_empty() {
|
||||
// Soft-skip, not a hard failure: serve /health, report Degraded.
|
||||
warn!(
|
||||
target: "meta-feeder",
|
||||
"usenet: no nntmux database url configured — upstream will soft-skip \
|
||||
(set it in the feeder's config page, or seed NNTMUX_DB_URL)"
|
||||
);
|
||||
self.settings = settings;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Connect eagerly so a bad url is visible at boot rather than on the
|
||||
// first query, but never fatally — the sidecar may still be starting.
|
||||
let url = settings.db_url.clone();
|
||||
let slot = self.db.clone();
|
||||
tokio::spawn(async move {
|
||||
match Db::connect(&url).await {
|
||||
Ok(db) => {
|
||||
match db.release_count().await {
|
||||
Ok(0) => warn!(
|
||||
target: "meta-feeder",
|
||||
"usenet: nntmux catalog is EMPTY — check the scan actually ran \
|
||||
(PATH_TO_NZBS must be non-empty, and groups need active=1)"
|
||||
),
|
||||
Ok(n) => info!(target: "meta-feeder", releases = n, "usenet: nntmux catalog connected"),
|
||||
Err(e) => warn!(target: "meta-feeder", error = %e, "usenet: catalog count failed"),
|
||||
}
|
||||
*slot.write().await = Some(db);
|
||||
}
|
||||
Err(e) => warn!(target: "meta-feeder", error = %e, "usenet: nntmux connect failed; upstream soft-skips"),
|
||||
}
|
||||
});
|
||||
self.settings = settings;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_query(
|
||||
&self,
|
||||
query: &GatewayQuery,
|
||||
max_results: usize,
|
||||
) -> Result<Vec<DiscoveryRecord>, GatewayError> {
|
||||
// `free_text` is the bare-word leaves only — the structured filters
|
||||
// (contentKind:, fileType:) are the gateway's routing gate, not a search
|
||||
// term, and passing them to a LIKE would match nothing.
|
||||
let term = query.free_text.clone();
|
||||
if term.trim().is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let db = self.db().await?;
|
||||
let releases = db
|
||||
.search(term.trim(), max_results)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux search: {e}")))?;
|
||||
|
||||
// Mint each hit's cid here, from its on-disk manifest — see the
|
||||
// `record_for` doc for why this cannot wait for compute. A release
|
||||
// whose manifest isn't written yet is DROPPED rather than surfaced
|
||||
// uncid'd: an unplayable row in the client's "raw sources" list is
|
||||
// worse than one fewer row.
|
||||
let mut out = Vec::with_capacity(releases.len());
|
||||
let mut skipped = 0usize;
|
||||
for r in &releases {
|
||||
match self.posting_cid(&r.guid) {
|
||||
Some(cid) => out.push(self.record_for(r, Some(&cid))),
|
||||
None => skipped += 1,
|
||||
}
|
||||
}
|
||||
debug!(
|
||||
target: "meta-feeder", %term, hits = out.len(), skipped,
|
||||
"usenet: catalog search"
|
||||
);
|
||||
if skipped > 0 {
|
||||
debug!(
|
||||
target: "meta-feeder", skipped,
|
||||
"usenet: releases skipped — no .nzb.gz on disk yet (scan still collating?)"
|
||||
);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
async fn compute_outcomes(&self, record_id: &str) -> Result<Vec<HashOutcome>, GatewayError> {
|
||||
let db = self.db().await?;
|
||||
let release = db
|
||||
.by_guid(record_id)
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux lookup: {e}")))?
|
||||
.ok_or(GatewayError::NotFound)?;
|
||||
|
||||
// The Message-IDs survive only in the on-disk `.nzb.gz` — nntmux purges
|
||||
// collections/binaries/parts once it has written the file.
|
||||
let manifest = nzb::load(&self.nzb_root, record_id)
|
||||
.map_err(|e| GatewayError::Permanent(format!("nzb read: {e}")))?
|
||||
.ok_or(GatewayError::NotFound)?;
|
||||
|
||||
let cid = compute_nzb_posting_cid(&manifest.message_ids);
|
||||
|
||||
// Same cid the search path already published for this release — the
|
||||
// client is committing to a cid it saw in the results, so the two must
|
||||
// agree. They do by construction: both mint from the same manifest.
|
||||
let mut record = self.record_for(&release, Some(&cid));
|
||||
// `manifest_url` is already set by `record_for` — see its doc for why
|
||||
// it must be on the search path, not only here.
|
||||
record
|
||||
.fields
|
||||
.insert("segmentCount".to_string(), manifest.message_ids.len().to_string());
|
||||
|
||||
Ok(vec![HashOutcome {
|
||||
hash: Hash(cid),
|
||||
hash_kind: HashKind::NzbPosting,
|
||||
// Metadata-only: we hold no media bytes. meta-share fetches the
|
||||
// articles from Usenet at playback.
|
||||
bytes: None,
|
||||
record: Some(record),
|
||||
file_extension: None,
|
||||
}])
|
||||
}
|
||||
|
||||
/// Serve a release's `.nzb` so the core can seed it. The cid segment is the
|
||||
/// release guid (optionally `.nzb`-suffixed so the core's extension sniffing
|
||||
/// names the stored blob correctly).
|
||||
async fn get_blob(&self, cid: &str) -> Option<Vec<u8>> {
|
||||
let guid = cid.strip_suffix(".nzb").unwrap_or(cid);
|
||||
match nzb::load(&self.nzb_root, guid) {
|
||||
Ok(Some(m)) => Some(m.xml),
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
warn!(target: "meta-feeder", %guid, error = %e, "usenet: blob read failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn health(&self) -> PluginHealth {
|
||||
if self.settings.db_url.is_empty() {
|
||||
return PluginHealth::Degraded {
|
||||
reason: "nntmux database not configured".into(),
|
||||
};
|
||||
}
|
||||
PluginHealth::Ok
|
||||
}
|
||||
|
||||
fn served_file_types(&self) -> &'static [&'static str] {
|
||||
&["video", "audio", "document", "archive"]
|
||||
}
|
||||
|
||||
fn served_content_kinds(&self) -> &'static [&'static str] {
|
||||
&["movie", "tv"]
|
||||
}
|
||||
|
||||
fn config_schema(&self) -> ConfigSchema {
|
||||
ConfigSchema {
|
||||
fields: vec![
|
||||
F::text("db_url", "nntmux database URL")
|
||||
.with_help(
|
||||
"MariaDB DSN of the nntmux sidecar's catalog, e.g. \
|
||||
mysql://nntmux:secret@nntmux-db:3306/nntmux. Blank → the \
|
||||
upstream soft-skips and this plugin reports Degraded. Takes \
|
||||
effect on the next feeder restart (no hot reload).",
|
||||
)
|
||||
.required(),
|
||||
F::text("nzb_root", "NZB store path")
|
||||
.with_help(
|
||||
"Path to nntmux's NZB directory as mounted INTO this container \
|
||||
— it must be the same volume nntmux's PATH_TO_NZBS points at. \
|
||||
Each release's ordered Message-IDs survive only in \
|
||||
<root>/<first-guid-char>/<guid>.nzb.gz, so a wrong path here \
|
||||
means every release resolves to NotFound.",
|
||||
)
|
||||
.required(),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn config_values(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"db_url": self.settings.db_url,
|
||||
"nzb_root": self.settings.nzb_root,
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user