Worph
@@ -0,0 +1,281 @@
|
||||
//! Read-only access to the nntmux sidecar's release catalog (MariaDB).
|
||||
//!
|
||||
//! We are a **consumer** of nntmux's schema, never a writer: the scanner owns
|
||||
//! that database, and everything in it is re-derivable by re-scanning. Treat it
|
||||
//! as persistent cache, not as a source of truth (the Tribler precedent — a
|
||||
//! stateful sidecar the gateway still sees as a stateless HTTP feeder).
|
||||
//!
|
||||
//! Only a narrow, long-stable column set is selected (`guid`, `searchname`,
|
||||
//! `name`, `size`, `postdate`, `categories_id`, `totalpart`). nntmux carries
|
||||
//! ~80 columns on `releases`, most of them scanner bookkeeping; binding a wide
|
||||
//! row here would make us fragile to upstream schema churn for no gain.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use sqlx::mysql::{MySqlPool, MySqlPoolOptions};
|
||||
use sqlx::Row;
|
||||
|
||||
/// One collated nntmux release — the metadata half of a Usenet posting.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Release {
|
||||
/// nntmux's public id, and the basename of the on-disk `<guid>.nzb.gz`.
|
||||
/// This is our `record_id`.
|
||||
pub guid: String,
|
||||
/// The collator's cleaned, scene-style name
|
||||
/// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`). Falls back to the
|
||||
/// raw posted `name` when the collator could not improve on it.
|
||||
pub search_name: String,
|
||||
/// Total posted size in bytes (RAR + par2 included, so an over-estimate of
|
||||
/// the final media file).
|
||||
pub size: u64,
|
||||
/// nntmux category id (`categories.id`). Coarse: 2000s = movies, 5000s = TV,
|
||||
/// 3000s = audio, 7000s = books — the Newznab category tree.
|
||||
pub category_id: i32,
|
||||
/// Unix seconds of the original posting date, when nntmux recorded one.
|
||||
pub post_date: Option<i64>,
|
||||
}
|
||||
|
||||
/// Connection pool to nntmux's MariaDB. Small by design — this feeder issues a
|
||||
/// handful of short SELECTs per query, and the scanner needs the headroom.
|
||||
#[derive(Clone)]
|
||||
pub struct Db {
|
||||
pool: MySqlPool,
|
||||
}
|
||||
|
||||
impl Db {
|
||||
/// Open a pool. Does **not** verify the schema — a feeder must come up and
|
||||
/// serve `/health` even when its upstream is unreachable, so the gateway's
|
||||
/// `depends_on` is satisfied and discovery does not soft-skip us forever.
|
||||
pub async fn connect(url: &str) -> Result<Self> {
|
||||
let pool = MySqlPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.acquire_timeout(std::time::Duration::from_secs(10))
|
||||
.connect(url)
|
||||
.await
|
||||
.with_context(|| "connect to nntmux mariadb")?;
|
||||
Ok(Self { pool })
|
||||
}
|
||||
|
||||
/// Free-text search over the collated release names.
|
||||
///
|
||||
/// `LIKE %term%` on `searchname` — deliberately simple. nntmux can be built
|
||||
/// with Manticore/Elasticsearch for real full-text search, but both are
|
||||
/// optional in the minimum stack and we do not want to require them. If
|
||||
/// search quality becomes the bottleneck, that is the upgrade path.
|
||||
pub async fn search(&self, term: &str, limit: usize) -> Result<Vec<Release>> {
|
||||
// ⚠ TOKENISE, don't match the phrase. Scene releases are dot-separated
|
||||
// (`Breaking.Bad.S01E01.1080p...`) while users type spaces ("breaking
|
||||
// bad"), so a single `LIKE %breaking bad%` matches essentially NOTHING
|
||||
// — the separator never lines up. Split into words and AND them, which
|
||||
// makes the separator irrelevant on both sides.
|
||||
//
|
||||
// Found by end-to-end search from meta-watch; the unit tests missed it
|
||||
// because they only exercised the escaping, never a realistic
|
||||
// scene-style name against a natural-language query.
|
||||
let words = search_tokens(term);
|
||||
if words.is_empty() {
|
||||
return Ok(vec![]);
|
||||
}
|
||||
let mut sql = String::from(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE ",
|
||||
);
|
||||
sql.push_str(
|
||||
&words
|
||||
.iter()
|
||||
.map(|_| "searchname LIKE ?")
|
||||
.collect::<Vec<_>>()
|
||||
.join(" AND "),
|
||||
);
|
||||
sql.push_str(" ORDER BY postdate DESC LIMIT ?");
|
||||
|
||||
let mut q = sqlx::query(&sql);
|
||||
for w in &words {
|
||||
q = q.bind(format!("%{}%", escape_like(w)));
|
||||
}
|
||||
let rows = q
|
||||
.bind(limit as i64)
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.context("query releases by name")?;
|
||||
Ok(rows.iter().map(row_to_release).collect())
|
||||
}
|
||||
|
||||
/// Look one release up by guid — the `compute_outcomes` path, where the
|
||||
/// record_id came back to us from the gateway.
|
||||
pub async fn by_guid(&self, guid: &str) -> Result<Option<Release>> {
|
||||
let row = sqlx::query(
|
||||
"SELECT guid, searchname, name, size, categories_id, \
|
||||
UNIX_TIMESTAMP(postdate) AS postdate_unix \
|
||||
FROM releases WHERE guid = ? LIMIT 1",
|
||||
)
|
||||
.bind(guid)
|
||||
.fetch_optional(&self.pool)
|
||||
.await
|
||||
.context("query release by guid")?;
|
||||
Ok(row.as_ref().map(row_to_release))
|
||||
}
|
||||
|
||||
/// Cheap liveness probe for `health()` — confirms the catalog is reachable
|
||||
/// *and* non-empty. An empty `releases` table is the visible symptom of both
|
||||
/// nntmux silent-zero traps (an unset `PATH_TO_NZBS`, or groups left
|
||||
/// `active=0`), so reporting it as degraded turns a silent failure into a
|
||||
/// legible one.
|
||||
pub async fn release_count(&self) -> Result<i64> {
|
||||
let row = sqlx::query("SELECT COUNT(*) AS n FROM releases")
|
||||
.fetch_one(&self.pool)
|
||||
.await
|
||||
.context("count releases")?;
|
||||
Ok(row.try_get::<i64, _>("n").unwrap_or(0))
|
||||
}
|
||||
}
|
||||
|
||||
fn row_to_release(row: &sqlx::mysql::MySqlRow) -> Release {
|
||||
let search_name: String = row
|
||||
.try_get::<String, _>("searchname")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.or_else(|| row.try_get::<String, _>("name").ok())
|
||||
.unwrap_or_default();
|
||||
Release {
|
||||
guid: row.try_get::<String, _>("guid").unwrap_or_default(),
|
||||
search_name,
|
||||
// nntmux stores size as an unsigned bigint; it arrives as u64 or
|
||||
// (older schemas) a decimal string.
|
||||
size: row
|
||||
.try_get::<u64, _>("size")
|
||||
.ok()
|
||||
.or_else(|| {
|
||||
row.try_get::<String, _>("size")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
})
|
||||
.unwrap_or(0),
|
||||
category_id: row.try_get::<i32, _>("categories_id").unwrap_or(0),
|
||||
post_date: row.try_get::<i64, _>("postdate_unix").ok(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a user query into match tokens.
|
||||
///
|
||||
/// Anything that is not alphanumeric is a separator — which folds the scene
|
||||
/// world's `.`/`_`/`-` and the human world's space onto the same footing, so
|
||||
/// "breaking bad", "Breaking.Bad" and "breaking-bad" all produce the same
|
||||
/// tokens and all match `Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE`.
|
||||
///
|
||||
/// Single characters are dropped: they match nearly every row and cost a full
|
||||
/// scan for no selectivity.
|
||||
fn search_tokens(term: &str) -> Vec<String> {
|
||||
term.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|w| w.chars().count() > 1)
|
||||
.map(|w| w.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Escape the `LIKE` wildcards so a user searching for `100%` does not match
|
||||
/// everything. Backslash is MySQL's default escape character.
|
||||
fn escape_like(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('%', "\\%")
|
||||
.replace('_', "\\_")
|
||||
}
|
||||
|
||||
/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`.
|
||||
///
|
||||
/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s
|
||||
/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets
|
||||
/// `METADATA_KEYS.md` §1 defines.
|
||||
pub fn file_type_for_category(category_id: i32) -> &'static str {
|
||||
match category_id / 1000 {
|
||||
2 | 5 | 6 => "video",
|
||||
3 => "audio",
|
||||
7 => "document",
|
||||
_ => "archive",
|
||||
}
|
||||
}
|
||||
|
||||
/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where
|
||||
/// the category tree is unambiguous — a wrong `contentKind` is worse than none,
|
||||
/// because meta-watch's curated rows gate on it.
|
||||
pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> {
|
||||
match category_id / 1000 {
|
||||
2 => Some("movie"),
|
||||
5 => Some("tv"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// ⚠ THE DOT-VS-SPACE REGRESSION. Scene releases are dot-separated and
|
||||
/// users type spaces; matching the raw phrase finds nothing. Caught by an
|
||||
/// end-to-end search from meta-watch that returned 0 results against a
|
||||
/// catalog that plainly contained the show.
|
||||
#[test]
|
||||
fn a_spaced_query_tokenises_to_match_a_dotted_scene_name() {
|
||||
let tokens = search_tokens("breaking bad");
|
||||
assert_eq!(tokens, vec!["breaking", "bad"]);
|
||||
|
||||
// Every token must be a substring of the real scene name — that is
|
||||
// exactly what the AND-ed LIKEs check.
|
||||
let scene = "Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE".to_lowercase();
|
||||
assert!(
|
||||
tokens.iter().all(|t| scene.contains(&t.to_lowercase())),
|
||||
"tokens {tokens:?} must all appear in {scene}"
|
||||
);
|
||||
|
||||
// The naive whole-phrase form does NOT — this is the bug, pinned.
|
||||
assert!(
|
||||
!scene.contains("breaking bad"),
|
||||
"if this ever passes, the dotted-name assumption changed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Separators are interchangeable in both directions.
|
||||
#[test]
|
||||
fn scene_and_human_separators_produce_the_same_tokens() {
|
||||
let expect = vec!["breaking".to_string(), "bad".to_string()];
|
||||
assert_eq!(search_tokens("breaking bad"), expect);
|
||||
assert_eq!(search_tokens("Breaking.Bad").iter().map(|s| s.to_lowercase()).collect::<Vec<_>>(), expect);
|
||||
assert_eq!(search_tokens("breaking-bad"), expect);
|
||||
assert_eq!(search_tokens("breaking_bad"), expect);
|
||||
assert_eq!(search_tokens(" breaking bad "), expect);
|
||||
}
|
||||
|
||||
/// Single characters are dropped (no selectivity, full scan) and an
|
||||
/// all-noise query yields nothing rather than matching the whole catalog.
|
||||
#[test]
|
||||
fn noise_only_queries_yield_no_tokens() {
|
||||
assert!(search_tokens("").is_empty());
|
||||
assert!(search_tokens(" ").is_empty());
|
||||
assert!(search_tokens("- . _").is_empty());
|
||||
assert_eq!(search_tokens("a bc"), vec!["bc"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn like_wildcards_are_escaped() {
|
||||
assert_eq!(escape_like("100%"), "100\\%");
|
||||
assert_eq!(escape_like("a_b"), "a\\_b");
|
||||
assert_eq!(escape_like("c:\\d"), "c:\\\\d");
|
||||
assert_eq!(escape_like("naruto"), "naruto");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn newznab_categories_map_to_file_types() {
|
||||
assert_eq!(file_type_for_category(2040), "video"); // movies HD
|
||||
assert_eq!(file_type_for_category(5040), "video"); // tv HD
|
||||
assert_eq!(file_type_for_category(3010), "audio");
|
||||
assert_eq!(file_type_for_category(7020), "document"); // ebook
|
||||
assert_eq!(file_type_for_category(4000), "archive"); // pc
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_kind_only_where_unambiguous() {
|
||||
assert_eq!(content_kind_for_category(2040), Some("movie"));
|
||||
assert_eq!(content_kind_for_category(5040), Some("tv"));
|
||||
assert_eq!(content_kind_for_category(3010), None);
|
||||
assert_eq!(content_kind_for_category(7020), None);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user