1.1.0: reach nntmux over its Newznab API (search + getnzb, metamesh key) with a per-release .nzb cache; drop the MariaDB/storage coupling
Publish image / publish (push) Failing after 13s
Publish image / publish (push) Failing after 13s
This commit is contained in:
Worph
@@ -0,0 +1,386 @@
|
||||
//! Client for **our own** nntmux, over its Newznab HTTP API — the only way this
|
||||
//! feeder reaches it.
|
||||
//!
|
||||
//! nntmux is a separate app (worph/AppStore `NNTmux`). Until 1.1 this feeder
|
||||
//! read nntmux's MariaDB with SQL and gunzipped `<guid>.nzb.gz` straight off
|
||||
//! its storage volume; that tied it to nntmux's schema, its DB password and its
|
||||
//! shard layout, across two apps. Now it uses what any Newznab client uses:
|
||||
//!
|
||||
//! | call | route | cost (nntmux per-user counters) |
|
||||
//! |---|---|---|
|
||||
//! | [`NntmuxApi::search`] | `GET {base}/api/v1/api?t=search&q=…&extended=1` | 1 `apirequests` |
|
||||
//! | [`NntmuxApi::details`] | `GET {base}/api/v1/api?t=details&id=<guid>` | 1 `apirequests` |
|
||||
//! | [`NntmuxApi::nzb`] | `GET {base}/getnzb?id=<guid>.nzb&r=<key>` | 1 `downloadrequests` |
|
||||
//! | [`NntmuxApi::caps`] | `GET {base}/api/v1/api?t=caps` | free, no key |
|
||||
//!
|
||||
//! ⚠ The API lives under **`/api/v1/api`** (Laravel's `api` prefix) — plain
|
||||
//! `/api` is a 404. `t=get` is not used: in this nntmux it *redirects* to
|
||||
//! `/getnzb` on the **public** `APP_URL` host, which a container on `pcs` should
|
||||
//! not round-trip through; `/getnzb` on the internal host is the same handler.
|
||||
//!
|
||||
//! ⚠ nntmux throttles the whole `/api` group to 60 requests/minute and caps
|
||||
//! each role per 24 h (`apirequests`, `downloadrequests`: Admin 1000, User 10).
|
||||
//! The key should belong to a dedicated user whose role has raised caps (the
|
||||
//! NNTmux app provisions `metamesh`); [`Pacer`] keeps us under the per-minute
|
||||
//! throttle, and the per-guid cache ([`super::cache`]) means each release's
|
||||
//! `.nzb` is downloaded once, ever.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use meta_feeder_sdk::types::GatewayError;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use super::catalog::Release;
|
||||
use crate::newznab::{classify, fetch_nzb, looks_like_html, newznab_error, status_error};
|
||||
|
||||
/// Default base URL: the NNTmux app's web container on the shared `pcs` network.
|
||||
pub const DEFAULT_NNTMUX_URL: &str = "http://nntmux";
|
||||
|
||||
/// Newznab's own page-size ceiling (`<limits max="100">`).
|
||||
pub const MAX_PAGE: usize = 100;
|
||||
|
||||
/// Requests allowed per [`PACE_WINDOW`] — under nntmux's 60/min throttle, with
|
||||
/// headroom for a human browsing nntmux with the same key.
|
||||
const PACE_BUDGET: usize = 50;
|
||||
const PACE_WINDOW: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Sliding-window limiter: at most [`PACE_BUDGET`] calls per [`PACE_WINDOW`].
|
||||
/// A cold search (one `t=search` + one `/getnzb` per new hit) bursts freely up
|
||||
/// to the budget, then waits instead of earning an HTTP 429.
|
||||
#[derive(Default)]
|
||||
pub struct Pacer {
|
||||
sent: Mutex<VecDeque<Instant>>,
|
||||
}
|
||||
|
||||
impl Pacer {
|
||||
pub async fn wait(&self) {
|
||||
loop {
|
||||
let mut sent = self.sent.lock().await;
|
||||
let now = Instant::now();
|
||||
while sent.front().is_some_and(|t| now.duration_since(*t) >= PACE_WINDOW) {
|
||||
sent.pop_front();
|
||||
}
|
||||
if sent.len() < PACE_BUDGET {
|
||||
sent.push_back(now);
|
||||
return;
|
||||
}
|
||||
let wait = PACE_WINDOW - now.duration_since(*sent.front().unwrap());
|
||||
drop(sent);
|
||||
tokio::time::sleep(wait).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct NntmuxApi {
|
||||
http: reqwest::Client,
|
||||
/// Scheme + authority (+ optional path prefix), no trailing slash.
|
||||
base: String,
|
||||
key: String,
|
||||
pacer: Pacer,
|
||||
}
|
||||
|
||||
impl NntmuxApi {
|
||||
pub fn new(http: reqwest::Client, base: &str, key: &str) -> Self {
|
||||
let base = base.trim().trim_end_matches('/');
|
||||
let base = if base.starts_with("http://") || base.starts_with("https://") {
|
||||
base.to_string()
|
||||
} else {
|
||||
format!("http://{base}")
|
||||
};
|
||||
Self {
|
||||
http,
|
||||
base,
|
||||
key: key.trim().to_string(),
|
||||
pacer: Pacer::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn base(&self) -> &str {
|
||||
&self.base
|
||||
}
|
||||
|
||||
fn api_url(&self) -> String {
|
||||
format!("{}/api/v1/api", self.base)
|
||||
}
|
||||
|
||||
/// Free-text search, newest first. `q` is passed as typed — nntmux tokenises
|
||||
/// it (Manticore full-text when present, SQL otherwise).
|
||||
pub async fn search(&self, q: &str, limit: usize) -> Result<Vec<Release>, GatewayError> {
|
||||
let limit = limit.clamp(1, MAX_PAGE).to_string();
|
||||
let xml = self
|
||||
.get_xml(&[("t", "search"), ("q", q), ("limit", &limit), ("extended", "1")])
|
||||
.await?;
|
||||
parse_items(&xml).map_err(|e| GatewayError::Transient(format!("nntmux search answer: {e}")))
|
||||
}
|
||||
|
||||
/// One release's metadata, by guid (`None` = nntmux doesn't have it).
|
||||
pub async fn details(&self, guid: &str) -> Result<Option<Release>, GatewayError> {
|
||||
let xml = match self.get_xml(&[("t", "details"), ("id", guid), ("extended", "1")]).await {
|
||||
Err(GatewayError::NotFound) => return Ok(None),
|
||||
other => other?,
|
||||
};
|
||||
let items = parse_items(&xml)
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux details answer: {e}")))?;
|
||||
Ok(items.into_iter().find(|r| r.guid == guid))
|
||||
}
|
||||
|
||||
/// The release's `.nzb` (plain XML). Spends one `downloadrequests` unit —
|
||||
/// callers go through the cache.
|
||||
pub async fn nzb(&self, guid: &str) -> Result<Vec<u8>, GatewayError> {
|
||||
self.pacer.wait().await;
|
||||
let url = format!("{}/getnzb?id={guid}.nzb&r={}", self.base, self.key);
|
||||
fetch_nzb(&self.http, &url, &self.base).await
|
||||
}
|
||||
|
||||
/// Reachability probe (`t=caps` needs no key and counts against nothing).
|
||||
pub async fn caps(&self) -> Result<(), GatewayError> {
|
||||
let url = format!("{}?t=caps", self.api_url());
|
||||
let resp = self
|
||||
.http
|
||||
.get(&url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux {} unreachable: {}", self.base, e.without_url())))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(status_error(&resp, &self.base));
|
||||
}
|
||||
let body = resp
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux caps: {}", e.without_url())))?;
|
||||
if body.contains("<caps") {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(GatewayError::Permanent(format!(
|
||||
"{} did not answer t=caps like a Newznab server — is this the nntmux base URL?",
|
||||
self.base
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
/// GET the API with the key; Newznab errors (answered as HTTP 200) are mapped
|
||||
/// through [`classify`]. The URL is never echoed — it carries the key.
|
||||
async fn get_xml(&self, params: &[(&str, &str)]) -> Result<String, GatewayError> {
|
||||
self.pacer.wait().await;
|
||||
let mut query: Vec<(&str, &str)> = params.to_vec();
|
||||
query.push(("apikey", &self.key));
|
||||
let resp = self
|
||||
.http
|
||||
.get(self.api_url())
|
||||
.query(&query)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux {}: {}", self.base, e.without_url())))?;
|
||||
if !resp.status().is_success() {
|
||||
return Err(status_error(&resp, &self.base));
|
||||
}
|
||||
let body = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| GatewayError::Transient(format!("nntmux {}: {}", self.base, e.without_url())))?;
|
||||
if looks_like_html(&body) {
|
||||
return Err(GatewayError::Permanent(format!(
|
||||
"{} answered the Newznab API with an HTML page — check the nntmux base URL",
|
||||
self.base
|
||||
)));
|
||||
}
|
||||
let text = String::from_utf8_lossy(&body).into_owned();
|
||||
if let Some(err) = newznab_error(&text) {
|
||||
return Err(classify(&err));
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a Newznab RSS answer into releases. Items without a guid are skipped.
|
||||
pub fn parse_items(xml: &str) -> anyhow::Result<Vec<Release>> {
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Item {
|
||||
title: String,
|
||||
guid: String,
|
||||
size: u64,
|
||||
enclosure_len: u64,
|
||||
category: i32,
|
||||
usenetdate: Option<i64>,
|
||||
pubdate: Option<i64>,
|
||||
}
|
||||
|
||||
let mut reader = Reader::from_str(xml);
|
||||
reader.config_mut().trim_text(true);
|
||||
let mut out = Vec::new();
|
||||
let mut item: Option<Item> = None;
|
||||
let mut field: Option<&'static str> = None;
|
||||
|
||||
loop {
|
||||
match reader.read_event() {
|
||||
Ok(Event::Start(e)) => match e.local_name().as_ref() {
|
||||
b"item" => item = Some(Item::default()),
|
||||
b"title" if item.is_some() => field = Some("title"),
|
||||
b"guid" if item.is_some() => field = Some("guid"),
|
||||
b"pubDate" if item.is_some() => field = Some("pubDate"),
|
||||
_ => field = None,
|
||||
},
|
||||
Ok(Event::Empty(e)) => {
|
||||
let Some(it) = item.as_mut() else { continue };
|
||||
let name = e.local_name();
|
||||
let attr = |k: &[u8]| -> Option<String> {
|
||||
e.attributes()
|
||||
.flatten()
|
||||
.find(|a| a.key.local_name().as_ref() == k)
|
||||
.and_then(|a| a.unescape_value().ok().map(|v| v.into_owned()))
|
||||
};
|
||||
match name.as_ref() {
|
||||
b"attr" => {
|
||||
let (Some(n), Some(v)) = (attr(b"name"), attr(b"value")) else { continue };
|
||||
match n.as_str() {
|
||||
// Several: the parent (2000) and the subcategory (2040).
|
||||
"category" => {
|
||||
if let Ok(c) = v.trim().parse::<i32>() {
|
||||
it.category = it.category.max(c);
|
||||
}
|
||||
}
|
||||
"size" => it.size = v.trim().parse().unwrap_or(0),
|
||||
"usenetdate" => it.usenetdate = rfc2822(&v),
|
||||
"guid" if it.guid.is_empty() => it.guid = v.trim().to_string(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
b"enclosure" => {
|
||||
it.enclosure_len = attr(b"length").and_then(|l| l.trim().parse().ok()).unwrap_or(0);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(t)) => {
|
||||
let (Some(it), Some(f)) = (item.as_mut(), field) else { continue };
|
||||
let v = t.unescape().map(|c| c.into_owned()).unwrap_or_default();
|
||||
match f {
|
||||
"title" => it.title = v,
|
||||
"guid" => it.guid = guid_of(&v),
|
||||
"pubDate" => it.pubdate = rfc2822(&v),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(Event::End(e)) => {
|
||||
if e.local_name().as_ref() == b"item" {
|
||||
if let Some(it) = item.take() {
|
||||
if !it.guid.is_empty() {
|
||||
out.push(Release {
|
||||
guid: it.guid,
|
||||
search_name: it.title,
|
||||
size: if it.size > 0 { it.size } else { it.enclosure_len },
|
||||
category_id: it.category,
|
||||
post_date: it.usenetdate.or(it.pubdate),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
field = None;
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(anyhow::anyhow!("newznab xml: {e}")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// nntmux's `<guid>` is a details URL (`{server}/details/<guid>`); the release
|
||||
/// guid is its last path segment. A bare guid passes through.
|
||||
fn guid_of(raw: &str) -> String {
|
||||
let t = raw.trim().trim_end_matches('/');
|
||||
t.rsplit('/').next().unwrap_or(t).to_string()
|
||||
}
|
||||
|
||||
fn rfc2822(s: &str) -> Option<i64> {
|
||||
chrono::DateTime::parse_from_rfc2822(s.trim()).ok().map(|d| d.timestamp())
|
||||
}
|
||||
|
||||
/// A release guid as nntmux mints it (hex/uuid): the only shape this feeder will
|
||||
/// put into a URL or a cache path. Guards `get_blob`, whose id comes off the wire.
|
||||
pub fn is_guid(s: &str) -> bool {
|
||||
!s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-')
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const SEARCH: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:newznab="http://www.newznab.com/DTD/2010/feeds/attributes/">
|
||||
<channel>
|
||||
<title>NNTmux</title>
|
||||
<newznab:response offset="0" total="2"/>
|
||||
<item>
|
||||
<title>Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD</title>
|
||||
<guid isPermaLink="true">http://localhost/details/97a2c0ffee1234</guid>
|
||||
<link>http://localhost/getnzb?id=97a2c0ffee1234.nzb&r=KEY</link>
|
||||
<pubDate>Sat, 03 Oct 2026 12:00:00 +0000</pubDate>
|
||||
<category>TV > HD</category>
|
||||
<enclosure url="http://localhost/getnzb?id=97a2c0ffee1234.nzb&r=KEY" length="734003200" type="application/x-nzb"/>
|
||||
<newznab:attr name="category" value="5000"/>
|
||||
<newznab:attr name="category" value="5040"/>
|
||||
<newznab:attr name="size" value="734003200"/>
|
||||
<newznab:attr name="usenetdate" value="Fri, 02 Oct 2026 08:30:00 +0000"/>
|
||||
</item>
|
||||
<item>
|
||||
<title>no-guid-item</title>
|
||||
</item>
|
||||
<item>
|
||||
<title>Some.Movie.2024.1080p</title>
|
||||
<guid>abcd-1234</guid>
|
||||
<enclosure url="x" length="42" type="application/x-nzb"/>
|
||||
<newznab:attr name="category" value="2040"/>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>"#;
|
||||
|
||||
#[test]
|
||||
fn parses_items_guids_categories_sizes_and_dates() {
|
||||
let r = parse_items(SEARCH).unwrap();
|
||||
assert_eq!(r.len(), 2, "an item without a guid is skipped");
|
||||
assert_eq!(r[0].guid, "97a2c0ffee1234");
|
||||
assert_eq!(r[0].search_name, "Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD");
|
||||
assert_eq!(r[0].category_id, 5040, "the most specific category wins");
|
||||
assert_eq!(r[0].size, 734_003_200);
|
||||
assert_eq!(r[0].post_date, Some(1_790_929_800), "usenetdate beats pubDate");
|
||||
assert_eq!(r[1].guid, "abcd-1234");
|
||||
assert_eq!(r[1].size, 42, "enclosure length is the size fallback");
|
||||
assert_eq!(r[1].post_date, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_channel_is_no_releases() {
|
||||
assert!(parse_items("<rss><channel></channel></rss>").unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guid_guard() {
|
||||
assert!(is_guid("97a2c0ffee1234"));
|
||||
assert!(is_guid("0d1b2c3d-aaaa-bbbb-cccc-000000000000"));
|
||||
assert!(!is_guid("../etc/passwd"));
|
||||
assert!(!is_guid("a/b"));
|
||||
assert!(!is_guid(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_urls_are_normalised() {
|
||||
let c = reqwest::Client::new();
|
||||
assert_eq!(NntmuxApi::new(c.clone(), "nntmux", "k").base(), "http://nntmux");
|
||||
assert_eq!(NntmuxApi::new(c, " http://nntmux:80/ ", "k").base(), "http://nntmux:80");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_pacer_lets_a_burst_through() {
|
||||
let p = Pacer::default();
|
||||
let t = Instant::now();
|
||||
for _ in 0..PACE_BUDGET {
|
||||
p.wait().await;
|
||||
}
|
||||
assert!(t.elapsed() < Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
//! Per-release `.nzb` cache, so each nntmux release is downloaded **once**.
|
||||
//!
|
||||
//! Every search hit must carry its portable `nzb-posting` cid (see
|
||||
//! `usenet::record_for`), and that cid is a digest over the `.nzb`'s
|
||||
//! Message-IDs — so a hit costs one `/getnzb` the first time it is seen. nntmux
|
||||
//! counts each against the key's daily `downloadrequests`; without this cache a
|
||||
//! popular term would re-download the same postings on every query. A collated
|
||||
//! nntmux release never changes, so an entry never goes stale.
|
||||
//!
|
||||
//! Layout: `<feeder state>/nntmux-nzb/<guid>.nzb` (plain XML, written
|
||||
//! atomically). The same bytes are what `get_blob` serves the gateway core for
|
||||
//! seeding. An in-memory map remembers the minted cid so a cached hit is not
|
||||
//! even re-parsed.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use meta_feeder_sdk::hash::compute_nzb_posting_cid;
|
||||
|
||||
use super::api::is_guid;
|
||||
use super::nzb::extract_message_ids;
|
||||
|
||||
/// Upper bound on the in-memory cid memo (the files themselves are unbounded).
|
||||
const MEMO_MAX: usize = 50_000;
|
||||
|
||||
/// A release's minted identity.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Posting {
|
||||
pub cid: String,
|
||||
pub segment_count: usize,
|
||||
}
|
||||
|
||||
pub struct NzbCache {
|
||||
dir: PathBuf,
|
||||
memo: Mutex<HashMap<String, Posting>>,
|
||||
}
|
||||
|
||||
impl NzbCache {
|
||||
pub fn new(dir: &Path) -> Self {
|
||||
Self {
|
||||
dir: dir.to_path_buf(),
|
||||
memo: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn path(&self, guid: &str) -> Option<PathBuf> {
|
||||
is_guid(guid).then(|| self.dir.join(format!("{guid}.nzb")))
|
||||
}
|
||||
|
||||
/// The cached `.nzb`, if any.
|
||||
pub fn get(&self, guid: &str) -> Option<Vec<u8>> {
|
||||
std::fs::read(self.path(guid)?).ok()
|
||||
}
|
||||
|
||||
/// Store a downloaded `.nzb` (tmp + rename, so a reader never sees half a file).
|
||||
pub fn put(&self, guid: &str, xml: &[u8]) -> std::io::Result<()> {
|
||||
let Some(path) = self.path(guid) else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "not a release guid"));
|
||||
};
|
||||
std::fs::create_dir_all(&self.dir)?;
|
||||
let tmp = path.with_extension("nzb.tmp");
|
||||
std::fs::write(&tmp, xml)?;
|
||||
std::fs::rename(&tmp, &path)
|
||||
}
|
||||
|
||||
/// Mint (or recall) the posting identity for `guid` from its `.nzb`.
|
||||
/// `None` when the document has no segments or doesn't parse.
|
||||
pub fn posting(&self, guid: &str, xml: &[u8]) -> Option<Posting> {
|
||||
if let Some(p) = self.memo.lock().unwrap().get(guid) {
|
||||
return Some(p.clone());
|
||||
}
|
||||
let ids = extract_message_ids(xml).ok()?;
|
||||
if ids.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let p = Posting {
|
||||
cid: compute_nzb_posting_cid(&ids),
|
||||
segment_count: ids.len(),
|
||||
};
|
||||
let mut memo = self.memo.lock().unwrap();
|
||||
if memo.len() >= MEMO_MAX {
|
||||
memo.clear();
|
||||
}
|
||||
memo.insert(guid.to_string(), p.clone());
|
||||
Some(p)
|
||||
}
|
||||
|
||||
/// The memoised identity only (no file read) — the search hot path.
|
||||
pub fn known(&self, guid: &str) -> Option<Posting> {
|
||||
self.memo.lock().unwrap().get(guid).cloned()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const NZB: &[u8] = br#"<?xml version="1.0"?><nzb><file><segments>
|
||||
<segment bytes="1" number="1">a@b</segment><segment bytes="1" number="2">c@d</segment>
|
||||
</segments></file></nzb>"#;
|
||||
|
||||
#[test]
|
||||
fn put_get_and_mint() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let c = NzbCache::new(&dir.path().join("nntmux-nzb"));
|
||||
assert!(c.get("abc123").is_none());
|
||||
c.put("abc123", NZB).unwrap();
|
||||
assert_eq!(c.get("abc123").unwrap(), NZB);
|
||||
let p = c.posting("abc123", NZB).unwrap();
|
||||
assert_eq!(p.segment_count, 2);
|
||||
assert_eq!(p.cid, compute_nzb_posting_cid(&["a@b".to_string(), "c@d".to_string()]));
|
||||
assert_eq!(c.known("abc123"), Some(p));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hostile_guid_never_becomes_a_path() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let c = NzbCache::new(dir.path());
|
||||
assert!(c.put("../escape", NZB).is_err());
|
||||
assert!(c.get("../escape").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_segmentless_nzb_mints_nothing() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let c = NzbCache::new(dir.path());
|
||||
assert!(c.posting("g", b"<nzb></nzb>").is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
//! An nntmux release as the feeder sees it, and the Newznab category mapping.
|
||||
//!
|
||||
//! Filled from nntmux's Newznab search answer ([`super::api`]); nothing here
|
||||
//! knows how it was fetched.
|
||||
|
||||
/// One collated nntmux release — the metadata half of a Usenet posting.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Release {
|
||||
/// nntmux's public id (the last path segment of the item's `<guid>`). This
|
||||
/// is our `record_id`, and the id `/getnzb` takes.
|
||||
pub guid: String,
|
||||
/// The collator's cleaned, scene-style name
|
||||
/// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`) — the item `<title>`.
|
||||
pub search_name: String,
|
||||
/// Total posted size in bytes (RAR + par2 included, so an over-estimate of
|
||||
/// the final media file). `0` when the answer carried none.
|
||||
pub size: u64,
|
||||
/// nntmux category id — the most specific `newznab:attr name="category"`.
|
||||
/// Coarse: 2000s = movies, 5000s = TV, 3000s = audio, 7000s = books.
|
||||
pub category_id: i32,
|
||||
/// Unix seconds of the original posting date (`usenetdate`), else the
|
||||
/// item's `pubDate`, when either parses.
|
||||
pub post_date: Option<i64>,
|
||||
}
|
||||
|
||||
/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`.
|
||||
///
|
||||
/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s
|
||||
/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets
|
||||
/// `METADATA_KEYS.md` §1 defines.
|
||||
pub fn file_type_for_category(category_id: i32) -> &'static str {
|
||||
match category_id / 1000 {
|
||||
2 | 5 | 6 => "video",
|
||||
3 => "audio",
|
||||
7 => "document",
|
||||
_ => "archive",
|
||||
}
|
||||
}
|
||||
|
||||
/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where
|
||||
/// the category tree is unambiguous — a wrong `contentKind` is worse than none,
|
||||
/// because meta-watch's curated rows gate on it.
|
||||
pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> {
|
||||
match category_id / 1000 {
|
||||
2 => Some("movie"),
|
||||
// ⚠ `episode`, not `tv`. `tv` is not a `contentKind` at all — it was a
|
||||
// `domain` value, and it isn't even that any more (film + tv merged
|
||||
// into `screen`, METADATA_KEYS.md §14.17). A kind outside the registry
|
||||
// vocabulary resolves to no domain and no workForm, so the row is
|
||||
// stamped with neither and is invisible to every client wall.
|
||||
5 => Some("episode"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn newznab_categories_map_to_file_types() {
|
||||
assert_eq!(file_type_for_category(2040), "video"); // movies HD
|
||||
assert_eq!(file_type_for_category(5040), "video"); // tv HD
|
||||
assert_eq!(file_type_for_category(3010), "audio");
|
||||
assert_eq!(file_type_for_category(7020), "document"); // ebook
|
||||
assert_eq!(file_type_for_category(4000), "archive"); // pc
|
||||
assert_eq!(file_type_for_category(10), "archive"); // nntmux "Other > Misc"
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_kind_only_where_unambiguous() {
|
||||
assert_eq!(content_kind_for_category(2040), Some("movie"));
|
||||
assert_eq!(content_kind_for_category(5040), Some("episode"));
|
||||
assert_eq!(content_kind_for_category(3010), None);
|
||||
assert_eq!(content_kind_for_category(7020), None);
|
||||
}
|
||||
}
|
||||
@@ -1,286 +0,0 @@
|
||||
//! Read-only access to the nntmux sidecar's release catalog (MariaDB).
|
||||
//!
|
||||
//! We are a **consumer** of nntmux's schema, never a writer: the scanner owns
|
||||
//! that database, and everything in it is re-derivable by re-scanning. Treat it
|
||||
//! as persistent cache, not as a source of truth (the Tribler precedent — a
|
||||
//! stateful sidecar the gateway still sees as a stateless HTTP feeder).
|
||||
//!
|
||||
//! Only a narrow, long-stable column set is selected (`guid`, `searchname`,
|
||||
//! `name`, `size`, `postdate`, `categories_id`, `totalpart`). nntmux carries
|
||||
//! ~80 columns on `releases`, most of them scanner bookkeeping; binding a wide
|
||||
//! row here would make us fragile to upstream schema churn for no gain.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions};
|
||||
use sqlx::Row;
|
||||
|
||||
/// One collated nntmux release — the metadata half of a Usenet posting.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Release {
|
||||
/// nntmux's public id, and the basename of the on-disk `<guid>.nzb.gz`.
|
||||
/// This is our `record_id`.
|
||||
pub guid: String,
|
||||
/// The collator's cleaned, scene-style name
|
||||
/// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`). Falls back to the
|
||||
/// raw posted `name` when the collator could not improve on it.
|
||||
pub search_name: String,
|
||||
/// Total posted size in bytes (RAR + par2 included, so an over-estimate of
|
||||
/// the final media file).
|
||||
pub size: u64,
|
||||
/// nntmux category id (`categories.id`). Coarse: 2000s = movies, 5000s = TV,
|
||||
/// 3000s = audio, 7000s = books — the Newznab category tree.
|
||||
pub category_id: i32,
|
||||
/// Unix seconds of the original posting date, when nntmux recorded one.
|
||||
pub post_date: Option<i64>,
|
||||
}
|
||||
|
||||
/// Connection pool to nntmux's MariaDB. Small by design — this feeder issues a
|
||||
/// handful of short SELECTs per query, and the scanner needs the headroom.
|
||||
#[derive(Clone)]
|
||||
pub struct Db {
|
||||
pool: MySqlPool,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Open a pool. Does **not** verify the schema — a feeder must come up and
|
||||
/// serve `/health` even when its upstream is unreachable, so the gateway's
|
||||
/// `depends_on` is satisfied and discovery does not soft-skip us forever.
|
||||
pub async fn connect(url: &str) -> Result<Self> {
|
||||
let pool = MySqlPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.acquire_timeout(std::time::Duration::from_secs(10))
|
||||
.connect(url)
|
||||
.await
|
||||
.with_context(|| "connect to nntmux mariadb")?;
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
/// Free-text search over the collated release names.
|
||||
///
|
||||
/// `LIKE %term%` on `searchname` — deliberately simple. nntmux can be built
|
||||
/// with Manticore/Elasticsearch for real full-text search, but both are
|
||||
/// optional in the minimum stack and we do not want to require them. If
|
||||
/// search quality becomes the bottleneck, that is the upgrade path.
|
||||
pub async fn search(&self, term: &str, limit: usize) -> Result<Vec<Release>> {
|
||||
// ⚠ TOKENISE, don't match the phrase. Scene releases are dot-separated
|
||||
// (`Breaking.Bad.S01E01.1080p...`) while users type spaces ("breaking
|
||||
// bad"), so a single `LIKE %breaking bad%` matches essentially NOTHING
|
||||
// — the separator never lines up. Split into words and AND them, which
|
||||
// makes the separator irrelevant on both sides.
|
||||
//
|
||||
// Found by end-to-end search from meta-watch; the unit tests missed it
|
||||
// because they only exercised the escaping, never a realistic
|
||||
// scene-style name against a natural-language query.
|
||||
let words = search_tokens(term);
|
||||
if words.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut sql = String::from(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE ",
|
||||
);
|
||||
sql.push_str(
|
||||
&words
|
||||
.iter()
|
||||
.map(|_| "searchname LIKE ?")
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND "),
|
||||
);
|
||||
sql.push_str(" ORDER BY postdate DESC LIMIT ?");
|
||||
|
||||
let mut q = sqlx::query(&sql);
|
||||
for w in &words {
|
||||
q = q.bind(format!("%{}%", escape_like(w)));
|
||||
}
|
||||
let rows = q
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.context("query releases by name")?;
|
||||
Ok(rows.iter().map(row_to_release).collect())
|
||||
}
|
||||
|
||||
/// Look one release up by guid — the `compute_outcomes` path, where the
|
||||
/// record_id came back to us from the gateway.
|
||||
pub async fn by_guid(&self, guid: &str) -> Result<Option<Release>> {
|
||||
let row = sqlx::query(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE guid = ? LIMIT 1",
|
||||
)
|
||||
.bind(guid)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.context("query release by guid")?;
|
||||
Ok(row.as_ref().map(row_to_release))
|
||||
}
|
||||
|
||||
/// Cheap liveness probe for `health()` — confirms the catalog is reachable
|
||||
/// *and* non-empty. An empty `releases` table is the visible symptom of both
|
||||
/// nntmux silent-zero traps (an unset `PATH_TO_NZBS`, or groups left
|
||||
/// `active=0`), so reporting it as degraded turns a silent failure into a
|
||||
/// legible one.
|
||||
pub async fn release_count(&self) -> Result<i64> {
|
||||
let row = sqlx::query("SELECT COUNT(*) AS n FROM releases")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.context("count releases")?;
|
||||
Ok(row.try_get::<i64, _>("n").unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
fn row_to_release(row: &sqlx::mysql::MySqlRow) -> Release {
|
||||
let search_name: String = row
|
||||
.try_get::<String, _>("searchname")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.or_else(|| row.try_get::<String, _>("name").ok())
|
||||
.unwrap_or_default();
|
||||
Release {
|
||||
guid: row.try_get::<String, _>("guid").unwrap_or_default(),
|
||||
search_name,
|
||||
// nntmux stores size as an unsigned bigint; it arrives as u64 or
|
||||
// (older schemas) a decimal string.
|
||||
size: row
|
||||
.try_get::<u64, _>("size")
|
||||
.ok()
|
||||
.or_else(|| {
|
||||
row.try_get::<String, _>("size")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
})
|
||||
.unwrap_or(0),
|
||||
category_id: row.try_get::<i32, _>("categories_id").unwrap_or(0),
|
||||
post_date: row.try_get::<i64, _>("postdate_unix").ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a user query into match tokens.
|
||||
///
|
||||
/// Anything that is not alphanumeric is a separator — which folds the scene
|
||||
/// world's `.`/`_`/`-` and the human world's space onto the same footing, so
|
||||
/// "breaking bad", "Breaking.Bad" and "breaking-bad" all produce the same
|
||||
/// tokens and all match `Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE`.
|
||||
///
|
||||
/// Single characters are dropped: they match nearly every row and cost a full
|
||||
/// scan for no selectivity.
|
||||
fn search_tokens(term: &str) -> Vec<String> {
|
||||
term.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| w.chars().count() > 1)
|
||||
.map(|w| w.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Escape the `LIKE` wildcards so a user searching for `100%` does not match
|
||||
/// everything. Backslash is MySQL's default escape character.
|
||||
fn escape_like(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_")
|
||||
}
|
||||
|
||||
/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`.
|
||||
///
|
||||
/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s
|
||||
/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets
|
||||
/// `METADATA_KEYS.md` §1 defines.
|
||||
pub fn file_type_for_category(category_id: i32) -> &'static str {
|
||||
match category_id / 1000 {
|
||||
2 | 5 | 6 => "video",
|
||||
3 => "audio",
|
||||
7 => "document",
|
||||
_ => "archive",
|
||||
}
|
||||
}
|
||||
|
||||
/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where
|
||||
/// the category tree is unambiguous — a wrong `contentKind` is worse than none,
|
||||
/// because meta-watch's curated rows gate on it.
|
||||
pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> {
|
||||
match category_id / 1000 {
|
||||
2 => Some("movie"),
|
||||
// ⚠ `episode`, not `tv`. `tv` is not a `contentKind` at all — it was a
|
||||
// `domain` value, and it isn't even that any more (film + tv merged
|
||||
// into `screen`, METADATA_KEYS.md §14.17). A kind outside the registry
|
||||
// vocabulary resolves to no domain and no workForm, so the row is
|
||||
// stamped with neither and is invisible to every client wall.
|
||||
5 => Some("episode"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// ⚠ THE DOT-VS-SPACE REGRESSION. Scene releases are dot-separated and
|
||||
/// users type spaces; matching the raw phrase finds nothing. Caught by an
|
||||
/// end-to-end search from meta-watch that returned 0 results against a
|
||||
/// catalog that plainly contained the show.
|
||||
#[test]
|
||||
fn a_spaced_query_tokenises_to_match_a_dotted_scene_name() {
|
||||
let tokens = search_tokens("breaking bad");
|
||||
assert_eq!(tokens, vec!["breaking", "bad"]);
|
||||
|
||||
// Every token must be a substring of the real scene name — that is
|
||||
// exactly what the AND-ed LIKEs check.
|
||||
let scene = "Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE".to_lowercase();
|
||||
assert!(
|
||||
tokens.iter().all(|t| scene.contains(&t.to_lowercase())),
|
||||
"tokens {tokens:?} must all appear in {scene}"
|
||||
);
|
||||
|
||||
// The naive whole-phrase form does NOT — this is the bug, pinned.
|
||||
assert!(
|
||||
!scene.contains("breaking bad"),
|
||||
"if this ever passes, the dotted-name assumption changed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Separators are interchangeable in both directions.
|
||||
#[test]
|
||||
fn scene_and_human_separators_produce_the_same_tokens() {
|
||||
let expect = vec!["breaking".to_string(), "bad".to_string()];
|
||||
assert_eq!(search_tokens("breaking bad"), expect);
|
||||
assert_eq!(search_tokens("Breaking.Bad").iter().map(|s| s.to_lowercase()).collect::<Vec<_>>(), expect);
|
||||
assert_eq!(search_tokens("breaking-bad"), expect);
|
||||
assert_eq!(search_tokens("breaking_bad"), expect);
|
||||
assert_eq!(search_tokens(" breaking bad "), expect);
|
||||
}
|
||||
|
||||
/// Single characters are dropped (no selectivity, full scan) and an
|
||||
/// all-noise query yields nothing rather than matching the whole catalog.
|
||||
#[test]
|
||||
fn noise_only_queries_yield_no_tokens() {
|
||||
assert!(search_tokens("").is_empty());
|
||||
assert!(search_tokens(" ").is_empty());
|
||||
assert!(search_tokens("- . _").is_empty());
|
||||
assert_eq!(search_tokens("a bc"), vec!["bc"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn like_wildcards_are_escaped() {
|
||||
assert_eq!(escape_like("100%"), "100\\%");
|
||||
assert_eq!(escape_like("a_b"), "a\\_b");
|
||||
assert_eq!(escape_like("c:\\d"), "c:\\\\d");
|
||||
assert_eq!(escape_like("naruto"), "naruto");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newznab_categories_map_to_file_types() {
|
||||
assert_eq!(file_type_for_category(2040), "video"); // movies HD
|
||||
assert_eq!(file_type_for_category(5040), "video"); // tv HD
|
||||
assert_eq!(file_type_for_category(3010), "audio");
|
||||
assert_eq!(file_type_for_category(7020), "document"); // ebook
|
||||
assert_eq!(file_type_for_category(4000), "archive"); // pc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_kind_only_where_unambiguous() {
|
||||
assert_eq!(content_kind_for_category(2040), Some("movie"));
|
||||
assert_eq!(content_kind_for_category(5040), Some("episode"));
|
||||
assert_eq!(content_kind_for_category(3010), None);
|
||||
assert_eq!(content_kind_for_category(7020), None);
|
||||
}
|
||||
}
|
||||
@@ -1,106 +1,16 @@
|
||||
//! Reading an nntmux release's manifest off disk and extracting its ordered
|
||||
//! article Message-IDs.
|
||||
//! Extracting an NZB's ordered article Message-IDs — the input of the portable
|
||||
//! `nzb-posting` (`0x1003`) cid.
|
||||
//!
|
||||
//! # Why the file and not the database
|
||||
//! # Why the `.nzb` and not nntmux's metadata
|
||||
//!
|
||||
//! You **cannot** read a finished release's Message-IDs out of nntmux's
|
||||
//! database. `NzbService::writeNzbForReleaseId()` writes the release's NZB and
|
||||
//! then *unconditionally* purges collections → binaries → parts (all FKs are
|
||||
//! `ON DELETE CASCADE`, and there is no retention setting). Those tables are
|
||||
//! transient staging for the header scan; for any *collated* release the
|
||||
//! ordered Message-IDs survive **only** in the on-disk gzip NZB at
|
||||
//! `<nzb_root>/<split>/<guid>.nzb.gz`.
|
||||
//!
|
||||
//! This was confirmed on the live path during the spike (the offline
|
||||
//! `import-nzbs` path never stages collections/binaries/parts at all, so it
|
||||
//! cannot be used to check this). It is the single load-bearing fact behind the
|
||||
//! whole feeder: lose it and you are back to grabbing `.nzb` files from an
|
||||
//! external indexer, which is exactly what this design exists to avoid.
|
||||
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
//! database: `NzbService::writeNzbForReleaseId()` writes the release's NZB and
|
||||
//! then purges collections → binaries → parts. For any *collated* release the
|
||||
//! ordered Message-IDs survive **only** in its `.nzb` — which this feeder now
|
||||
//! downloads over nntmux's Newznab API (`/getnzb`, see [`super::api`]) and
|
||||
//! caches ([`super::cache`]), instead of gunzipping it off nntmux's storage.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use flate2::read::GzDecoder;
|
||||
|
||||
/// A parsed nntmux manifest: the raw NZB XML plus the Message-IDs in posting
|
||||
/// order.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Manifest {
|
||||
/// The decompressed NZB XML. Handed to the gateway verbatim so it can seed
|
||||
/// it as an ordinary sha2-256 cid — the consumer parses this exact document.
|
||||
pub xml: Vec<u8>,
|
||||
/// Every `<segment>` Message-ID across every `<file>`, in document order.
|
||||
/// Bare form (no surrounding `<>`), matching how NZB stores them and how
|
||||
/// meta-share's `NzbSegment.message_id` holds them.
|
||||
pub message_ids: Vec<String>,
|
||||
}
|
||||
|
||||
/// How deep we are willing to probe for a shard. nntmux caps its own loop at
|
||||
/// 32, but every real deployment sits in the low single digits and a guid is a
|
||||
/// 36-char uuid; 8 covers any sane `nzbsplitlevel` while keeping the probe below.
|
||||
const MAX_SPLIT_LEVEL: usize = 8;
|
||||
|
||||
/// Build the path a release's `.nzb.gz` would occupy at an exact shard depth:
|
||||
/// one directory per leading guid character, mirroring nntmux's
|
||||
/// `NZB::buildNZBPath()` (`$nzbPath .= $releaseGuid[$i].'/'`).
|
||||
pub fn nzb_path_at(nzb_root: &Path, guid: &str, levels: usize) -> Option<PathBuf> {
|
||||
if guid.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut path = nzb_root.to_path_buf();
|
||||
for c in guid.chars().take(levels) {
|
||||
path.push(c.to_string());
|
||||
}
|
||||
Some(path.join(format!("{guid}.nzb.gz")))
|
||||
}
|
||||
|
||||
/// Locate a release's `.nzb.gz` in nntmux's sharded store.
|
||||
///
|
||||
/// **The shard depth is an nntmux *setting* (`settings.nzbsplitlevel`), not a
|
||||
/// constant.** The PHP code's own fallback is 1, but the maintainer image ships
|
||||
/// a database seeded with **4** (`storage/nzb/9/7/a/2/97a2….nzb.gz`) — so a
|
||||
/// hardcoded depth silently resolves to a path that does not exist, every
|
||||
/// release is skipped as "no .nzb.gz on disk yet", and the feeder reports zero
|
||||
/// hits while the scan is in fact working perfectly. An operator can also
|
||||
/// change the setting *after* files are written, leaving a store with mixed
|
||||
/// depths. So resolve by probing the candidate depths, deepest first.
|
||||
///
|
||||
/// Still not a glob: at most `MAX_SPLIT_LEVEL + 1` `stat` calls against exact
|
||||
/// paths, no directory enumeration — cheap enough for the compute path.
|
||||
pub fn nzb_path(nzb_root: &Path, guid: &str) -> Option<PathBuf> {
|
||||
let deepest = guid.chars().count().min(MAX_SPLIT_LEVEL);
|
||||
(0..=deepest)
|
||||
.rev()
|
||||
.filter_map(|levels| nzb_path_at(nzb_root, guid, levels))
|
||||
.find(|path| path.exists())
|
||||
}
|
||||
|
||||
/// Read + gunzip a release's `.nzb.gz` and pull out its Message-IDs.
|
||||
///
|
||||
/// Returns `Ok(None)` when the file is absent — a normal, non-fatal state: a
|
||||
/// release row can exist before `createNZBs()` has written its manifest, and
|
||||
/// the caller simply skips that record rather than failing the whole query.
|
||||
pub fn load(nzb_root: &Path, guid: &str) -> Result<Option<Manifest>> {
|
||||
let Some(path) = nzb_path(nzb_root, guid) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
let raw = std::fs::read(&path).with_context(|| format!("read nzb {}", path.display()))?;
|
||||
let mut xml = Vec::new();
|
||||
GzDecoder::new(&raw[..])
|
||||
.read_to_end(&mut xml)
|
||||
.with_context(|| format!("gunzip nzb {}", path.display()))?;
|
||||
|
||||
let message_ids = extract_message_ids(&xml)
|
||||
.with_context(|| format!("parse nzb {}", path.display()))?;
|
||||
if message_ids.is_empty() {
|
||||
anyhow::bail!("nzb {} has no segments", path.display());
|
||||
}
|
||||
Ok(Some(Manifest { xml, message_ids }))
|
||||
}
|
||||
|
||||
/// Pull every `<segment>` body out of an NZB document, in order.
|
||||
///
|
||||
@@ -110,7 +20,7 @@ pub fn load(nzb_root: &Path, guid: &str) -> Result<Option<Manifest>> {
|
||||
/// meta-share would happily play. Surrounding `<>` are stripped if a producer
|
||||
/// included them — the mint normalises this too, but keeping the stored form
|
||||
/// canonical means the two never disagree about what was hashed.
|
||||
pub(crate) fn extract_message_ids(xml: &[u8]) -> Result<Vec<String>> {
|
||||
pub fn extract_message_ids(xml: &[u8]) -> Result<Vec<String>> {
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader;
|
||||
|
||||
@@ -183,74 +93,4 @@ mod tests {
|
||||
let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse");
|
||||
assert!(ids.iter().all(|i| !i.contains('<') && !i.contains('>')));
|
||||
}
|
||||
|
||||
/// One directory per leading guid character, exactly as nntmux's
|
||||
/// `buildNZBPath()` writes them.
|
||||
#[test]
|
||||
fn nzb_path_at_shards_one_dir_per_leading_char() {
|
||||
let p = nzb_path_at(Path::new("/s/nzb"), "abc123", 4).expect("path");
|
||||
assert_eq!(p, Path::new("/s/nzb/a/b/c/1/abc123.nzb.gz"));
|
||||
let p0 = nzb_path_at(Path::new("/s/nzb"), "abc123", 0).expect("path");
|
||||
assert_eq!(p0, Path::new("/s/nzb/abc123.nzb.gz"));
|
||||
}
|
||||
|
||||
/// The regression this exists for: the maintainer image seeds
|
||||
/// `nzbsplitlevel = 4`, so a resolver hardcoded to depth 1 finds nothing and
|
||||
/// every real release is silently skipped as "no .nzb.gz on disk yet".
|
||||
#[test]
|
||||
fn nzb_path_finds_a_level_4_shard() {
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
let guid = "97a2218c-d59b-4bc0-bb67-8a7d41e93fb5";
|
||||
let deep = dir.path().join("9/7/a/2");
|
||||
std::fs::create_dir_all(&deep).expect("mkdir");
|
||||
std::fs::write(deep.join(format!("{guid}.nzb.gz")), b"x").expect("write");
|
||||
|
||||
let got = nzb_path(dir.path(), guid).expect("resolved");
|
||||
assert_eq!(got, deep.join(format!("{guid}.nzb.gz")));
|
||||
}
|
||||
|
||||
/// A store written at a shallower depth (or an operator who lowered
|
||||
/// `nzbsplitlevel` after the fact) must still resolve.
|
||||
#[test]
|
||||
fn nzb_path_finds_a_level_1_shard() {
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
std::fs::create_dir_all(dir.path().join("a")).expect("mkdir");
|
||||
std::fs::write(dir.path().join("a/abc123.nzb.gz"), b"x").expect("write");
|
||||
|
||||
let got = nzb_path(dir.path(), "abc123").expect("resolved");
|
||||
assert_eq!(got, dir.path().join("a/abc123.nzb.gz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nzb_path_is_none_when_absent_at_every_depth() {
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
assert!(nzb_path(dir.path(), "abc123").is_none());
|
||||
}
|
||||
|
||||
/// A release row can exist before `createNZBs()` has written the manifest.
|
||||
/// That is normal — the caller skips the record, it is not an error.
|
||||
#[test]
|
||||
fn missing_file_is_none_not_error() {
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
let got = load(dir.path(), "deadbeef").expect("no error");
|
||||
assert!(got.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_a_gzipped_nzb() {
|
||||
use flate2::write::GzEncoder;
|
||||
use flate2::Compression;
|
||||
use std::io::Write;
|
||||
|
||||
let dir = tempfile::tempdir().expect("tmp");
|
||||
std::fs::create_dir_all(dir.path().join("a")).expect("mkdir");
|
||||
let mut enc = GzEncoder::new(Vec::new(), Compression::default());
|
||||
enc.write_all(SAMPLE.as_bytes()).expect("gz write");
|
||||
let gz = enc.finish().expect("gz finish");
|
||||
std::fs::write(dir.path().join("a/abc.nzb.gz"), gz).expect("write");
|
||||
|
||||
let m = load(dir.path(), "abc").expect("load").expect("present");
|
||||
assert_eq!(m.message_ids.len(), 3);
|
||||
assert_eq!(m.xml, SAMPLE.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user