4 Commits
17 changed files with 1491 additions and 38 deletions
+21 -3
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "meta-feeder-sdk" name = "meta-feeder-sdk"
version = "1.7.0" version = "1.10.0"
edition = "2021" edition = "2021"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
@@ -37,11 +37,15 @@ default = ["feeder"]
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default # FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
# so existing consumers build exactly what they always did. # so existing consumers build exactly what they always did.
feeder = [ feeder = [
"beacon", "beacon", "filename", "claims",
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures", "dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes", "dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
"dep:regex", "dep:redb", "dep:redb",
] ]
# Release-name parsing (`src/filename_meta.rs`: clean_title, season/episode,
# quality, codec). Needs only `regex`, so a client (meta-watch) can derive a
# display name from a raw release string without the feeder machinery.
filename = ["dep:regex"]
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc, # Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
# so a meta-* service takes the SDK with `default-features = false, # so a meta-* service takes the SDK with `default-features = false,
# features = ["beacon"]` and pulls in none of the feeder machinery. # features = ["beacon"]` and pulls in none of the feeder machinery.
@@ -52,6 +56,15 @@ beacon = ["dep:libc"]
transport = ["feeder", "dep:cid", "dep:tokio-util"] transport = ["feeder", "dep:cid", "dep:tokio-util"]
# Conformance checks any transport implementation must pass. # Conformance checks any transport implementation must pass.
testkit = ["transport"] testkit = ["transport"]
# MMUR2 signed claims (`src/claims.rs`): the `SignedClaim` type, its canonical
# bytes, signing (k256, compact low-S), verification and the writer key
# (`METAMESH_SIGNING_KEY` / `_FILE`). `feeder` enables it because the gateway
# wire (`DiscoveryRecord`, `EnrichPatch`) carries claims; a meta-* service that
# only needs claims takes `default-features = false, features = ["claims"]`.
claims = [
"dep:k256", "dep:bs58", "dep:sha2", "dep:base64", "dep:rand_core",
"dep:anyhow", "dep:thiserror",
]
[dependencies] [dependencies]
# Async runtime + HTTP server (the serve_feeders harness) # Async runtime + HTTP server (the serve_feeders harness)
@@ -76,6 +89,11 @@ bytes = { version = "1", optional = true }
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). # Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
regex = { version = "1", optional = true } regex = { version = "1", optional = true }
# MMUR2 claims (the `claims` feature).
k256 = { version = "0.13", features = ["ecdsa", "std"], optional = true }
bs58 = { version = "0.5", optional = true }
rand_core = { version = "0.6", features = ["getrandom"], optional = true }
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see # Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
# cache.rs. Pure-Rust single-process embedded store. # cache.rs. Pure-Rust single-process embedded store.
redb = { version = "2", optional = true } redb = { version = "2", optional = true }
+602
View File
@@ -0,0 +1,602 @@
//! MMUR2 signed claims: signer `uid` says `key = value` about content `cid`, at
//! `ts` (milliseconds, also the version). See meta-root
//! `docs/implementation/metacore-claims-model.md` §3.
//!
//! Wire spec (pinned by `tests/claim-vectors.json`, vendored from meta-core's
//! `internal/claim/testdata` — `scripts/check-claim-vectors.sh` guards it):
//!
//! ```text
//! domain = b"MMUR2\n"
//! payload = canonical CBOR array [cid, uid, key, value, ts]
//! bytes_to_sign = domain || payload
//! sig = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S
//! ```
//!
//! `value` is CBOR text (public), bytes (private envelope) or null (retraction).
//! The encoder is hand-rolled: those five shapes are all MMUR2 needs, and a
//! hand-rolled head encoder cannot drift from "smallest integer form".
//!
//! The type and its JSON form are always available with the `feeder` feature
//! (the gateway wire carries claims); signing and verification are here too,
//! behind `claims`, which `feeder` enables.
use std::collections::BTreeMap;
use base64::Engine as _;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
/// Domain separator. Changing it invalidates every claim everywhere.
pub const DOMAIN: &[u8] = b"MMUR2\n";
/// A claim's value. The SQLite storage type in meta-core mirrors it.
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub enum ClaimValue {
/// Retraction (tombstone).
#[default]
Null,
/// Public plaintext.
Text(String),
/// Private envelope (XChaCha20-Poly1305), opaque to everyone but the owner.
Bytes(Vec<u8>),
}
impl ClaimValue {
pub fn is_null(&self) -> bool {
matches!(self, ClaimValue::Null)
}
pub fn as_text(&self) -> Option<&str> {
match self {
ClaimValue::Text(s) => Some(s),
_ => None,
}
}
}
/// One signed vote.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedClaim {
pub cid: String,
pub uid: String,
pub key: String,
#[serde(with = "value_json")]
pub value: ClaimValue,
pub ts: u64,
/// 64-byte compact r||s, base64 in JSON.
#[serde(with = "sig_b64")]
pub sig: Vec<u8>,
}
impl SignedClaim {
/// The exact bytes the signature covers.
pub fn signed_bytes(&self) -> Vec<u8> {
signed_bytes(&self.cid, &self.uid, &self.key, &self.value, self.ts)
}
}
/// `domain || canonical CBOR [cid, uid, key, value, ts]`.
pub fn signed_bytes(cid: &str, uid: &str, key: &str, value: &ClaimValue, ts: u64) -> Vec<u8> {
let extra = match value {
ClaimValue::Text(s) => s.len(),
ClaimValue::Bytes(b) => b.len(),
ClaimValue::Null => 0,
};
let mut b = Vec::with_capacity(DOMAIN.len() + cid.len() + uid.len() + key.len() + extra + 32);
b.extend_from_slice(DOMAIN);
head(&mut b, 4, 5);
text(&mut b, cid);
text(&mut b, uid);
text(&mut b, key);
match value {
ClaimValue::Text(s) => text(&mut b, s),
ClaimValue::Bytes(bytes) => {
head(&mut b, 2, bytes.len() as u64);
b.extend_from_slice(bytes);
}
ClaimValue::Null => b.push(0xf6),
}
head(&mut b, 0, ts);
b
}
fn text(b: &mut Vec<u8>, s: &str) {
head(b, 3, s.len() as u64);
b.extend_from_slice(s.as_bytes());
}
/// CBOR major type + argument, smallest form.
fn head(b: &mut Vec<u8>, major: u8, n: u64) {
let m = major << 5;
if n < 24 {
b.push(m | n as u8);
} else if n <= 0xff {
b.extend_from_slice(&[m | 24, n as u8]);
} else if n <= 0xffff {
b.push(m | 25);
b.extend_from_slice(&(n as u16).to_be_bytes());
} else if n <= 0xffff_ffff {
b.push(m | 26);
b.extend_from_slice(&(n as u32).to_be_bytes());
} else {
b.push(m | 27);
b.extend_from_slice(&n.to_be_bytes());
}
}
/// JSON (human-readable) uses the spec §3.4 form — a string, `{"b64": …}` or
/// `null`. Binary formats (the bincode gateway wire) use a plain tagged enum:
/// bincode cannot decode the untagged JSON shape.
mod value_json {
use super::*;
#[derive(Serialize, Deserialize)]
struct B64 {
b64: String,
}
#[derive(Serialize, Deserialize)]
enum Wire {
Null,
Text(String),
Bytes(Vec<u8>),
}
pub fn serialize<S: Serializer>(v: &ClaimValue, s: S) -> Result<S::Ok, S::Error> {
if !s.is_human_readable() {
let w = match v {
ClaimValue::Null => Wire::Null,
ClaimValue::Text(t) => Wire::Text(t.clone()),
ClaimValue::Bytes(b) => Wire::Bytes(b.clone()),
};
return w.serialize(s);
}
match v {
ClaimValue::Null => s.serialize_none(),
ClaimValue::Text(t) => s.serialize_str(t),
ClaimValue::Bytes(b) => B64 {
b64: base64::engine::general_purpose::STANDARD.encode(b),
}
.serialize(s),
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<ClaimValue, D::Error> {
if !d.is_human_readable() {
return Ok(match Wire::deserialize(d)? {
Wire::Null => ClaimValue::Null,
Wire::Text(t) => ClaimValue::Text(t),
Wire::Bytes(b) => ClaimValue::Bytes(b),
});
}
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
Text(String),
Bytes(B64),
}
Ok(match Option::<Raw>::deserialize(d)? {
None => ClaimValue::Null,
Some(Raw::Text(t)) => ClaimValue::Text(t),
Some(Raw::Bytes(B64 { b64 })) => ClaimValue::Bytes(
base64::engine::general_purpose::STANDARD
.decode(b64)
.map_err(de::Error::custom)?,
),
})
}
}
/// base64 in JSON, raw bytes in binary formats.
mod sig_b64 {
use super::*;
pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
if !s.is_human_readable() {
// A plain sequence, symmetric with `Vec::<u8>::deserialize` below.
return v.serialize(s);
}
s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(v))
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
if !d.is_human_readable() {
return Vec::<u8>::deserialize(d);
}
let s = String::deserialize(d)?;
base64::engine::general_purpose::STANDARD
.decode(s)
.map_err(de::Error::custom)
}
}
/// Union-flatten a claim list into a field map: every key that has at least
/// one non-retracted text vote, with the first such value seen. A
/// **superset view for filtering and scoring**, not a merge — the merge (most
/// signers → oldest ts → smallest value, own vote first) lives only in
/// meta-core (`POST /api/merge`). Ciphertext never appears.
pub fn union_fields(claims: &[SignedClaim]) -> BTreeMap<String, String> {
let mut out = BTreeMap::new();
for c in claims {
if let ClaimValue::Text(v) = &c.value {
out.entry(c.key.clone()).or_insert_with(|| v.clone());
}
}
out
}
#[cfg(feature = "claims")]
pub use crypto::*;
#[cfg(feature = "claims")]
mod crypto {
use super::*;
use k256::ecdsa::signature::hazmat::{PrehashSigner, PrehashVerifier};
use k256::ecdsa::{Signature, SigningKey, VerifyingKey};
use sha2::{Digest, Sha256};
/// Why a claim failed verification.
#[derive(Debug, thiserror::Error)]
pub enum ClaimError {
#[error("claim: missing cid or key")]
Shape,
#[error("uid: {0}")]
Uid(String),
#[error("sig: want a 64-byte compact signature")]
SigLen,
#[error("sig: high-S signatures are refused (MMUR2 is low-S only)")]
HighS,
#[error("sig: does not verify")]
Invalid,
}
/// Recover the verifying key a uid encodes (`"z" + base58btc(33-byte sec1)`).
pub fn pubkey_from_uid(uid: &str) -> Result<VerifyingKey, ClaimError> {
let body = uid
.strip_prefix('z')
.ok_or_else(|| ClaimError::Uid("missing multibase 'z' prefix".into()))?;
let raw = bs58::decode(body)
.into_vec()
.map_err(|e| ClaimError::Uid(e.to_string()))?;
if raw.len() != 33 {
return Err(ClaimError::Uid(format!(
"pubkey is {} bytes, want 33",
raw.len()
)));
}
VerifyingKey::from_sec1_bytes(&raw).map_err(|e| ClaimError::Uid(e.to_string()))
}
/// The uid of a verifying key.
pub fn uid_from_pubkey(vk: &VerifyingKey) -> String {
let point = vk.to_encoded_point(true);
format!("z{}", bs58::encode(point.as_bytes()).into_string())
}
impl SignedClaim {
/// Check shape, low-S and the signature against the uid's key.
pub fn verify(&self) -> Result<(), ClaimError> {
if self.cid.is_empty() || self.key.is_empty() {
return Err(ClaimError::Shape);
}
if self.sig.len() != 64 {
return Err(ClaimError::SigLen);
}
let sig = Signature::from_slice(&self.sig).map_err(|_| ClaimError::Invalid)?;
if sig.normalize_s().is_some() {
return Err(ClaimError::HighS);
}
let vk = pubkey_from_uid(&self.uid)?;
let digest = Sha256::digest(self.signed_bytes());
vk.verify_prehash(&digest, &sig)
.map_err(|_| ClaimError::Invalid)
}
}
/// A writer's identity: one key per writing container (spec D2).
#[derive(Clone)]
pub struct Signer {
key: SigningKey,
uid: String,
}
impl std::fmt::Debug for Signer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Signer")
.field("uid", &self.uid)
.finish_non_exhaustive()
}
}
impl Signer {
/// From a 32-byte secret, hex (optionally `0x`-prefixed).
pub fn from_hex(hex_key: &str) -> anyhow::Result<Self> {
let s = hex_key.trim();
let s = s
.strip_prefix("0x")
.or_else(|| s.strip_prefix("0X"))
.unwrap_or(s);
let raw = decode_hex(s).ok_or_else(|| anyhow::anyhow!("signing key is not hex"))?;
if raw.len() != 32 {
anyhow::bail!("signing key must be 32 bytes, got {}", raw.len());
}
let key = SigningKey::from_slice(&raw)
.map_err(|e| anyhow::anyhow!("invalid signing key: {e}"))?;
let uid = uid_from_pubkey(key.verifying_key());
Ok(Signer { key, uid })
}
/// A fresh random key.
pub fn generate() -> Self {
let key = SigningKey::random(&mut rand_core::OsRng);
let uid = uid_from_pubkey(key.verifying_key());
Signer { key, uid }
}
pub fn uid(&self) -> &str {
&self.uid
}
/// The secret as lowercase hex (for writing a generated key file).
pub fn secret_hex(&self) -> String {
self.key
.to_bytes()
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
/// Sign one claim at `ts`. RFC 6979 deterministic, normalised to low-S.
pub fn sign(&self, cid: &str, key: &str, value: ClaimValue, ts: u64) -> SignedClaim {
let bytes = signed_bytes(cid, &self.uid, key, &value, ts);
let digest = Sha256::digest(&bytes);
let sig: Signature = self
.key
.sign_prehash(&digest)
.expect("a 32-byte prehash always signs");
let sig = sig.normalize_s().unwrap_or(sig);
SignedClaim {
cid: cid.to_string(),
uid: self.uid.clone(),
key: key.to_string(),
value,
ts,
sig: sig.to_bytes().to_vec(),
}
}
/// Sign every field of a flat map as text claims on `cid`, all at `ts`.
pub fn sign_fields(
&self,
cid: &str,
fields: &BTreeMap<String, String>,
ts: u64,
) -> Vec<SignedClaim> {
fields
.iter()
.map(|(k, v)| self.sign(cid, k, ClaimValue::Text(v.clone()), ts))
.collect()
}
/// A signed retraction of `key` on `cid`.
pub fn retract(&self, cid: &str, key: &str, ts: u64) -> SignedClaim {
self.sign(cid, key, ClaimValue::Null, ts)
}
}
/// Milliseconds since the epoch — the `ts` of a claim signed now. Strictly
/// increasing per process: `ts` is the version, so two writes to one key in
/// the same millisecond must not make the second one `stale`.
pub fn now_ms() -> u64 {
use std::sync::atomic::{AtomicU64, Ordering};
static LAST: AtomicU64 = AtomicU64::new(0);
let wall = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let mut last = LAST.load(Ordering::Relaxed);
loop {
let next = wall.max(last + 1);
match LAST.compare_exchange_weak(last, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return next,
Err(seen) => last = seen,
}
}
}
/// Env var holding the signer's secret as hex.
pub const SIGNING_KEY_ENV: &str = "METAMESH_SIGNING_KEY";
/// Env var naming a key file (bind-mounted); generated on first boot.
pub const SIGNING_KEY_FILE_ENV: &str = "METAMESH_SIGNING_KEY_FILE";
/// The writer's key (spec §9): `METAMESH_SIGNING_KEY` (hex) wins; else
/// `METAMESH_SIGNING_KEY_FILE`, generated and written (mode 0600) when the
/// file does not exist yet. Neither set → error: a writer must never sign
/// with a silent ephemeral identity.
pub fn load_or_create_key() -> anyhow::Result<Signer> {
if let Some(hex) = env_nonempty(SIGNING_KEY_ENV) {
return Signer::from_hex(&hex);
}
let Some(path) = env_nonempty(SIGNING_KEY_FILE_ENV) else {
anyhow::bail!("no signing key: set {SIGNING_KEY_ENV} or {SIGNING_KEY_FILE_ENV}");
};
load_or_create_key_file(std::path::Path::new(&path))
}
/// Read a hex key file, or generate one there (mode 0600) if absent.
pub fn load_or_create_key_file(path: &std::path::Path) -> anyhow::Result<Signer> {
match std::fs::read_to_string(path) {
Ok(s) => Signer::from_hex(&s),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
let signer = Signer::generate();
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
write_secret(path, &signer.secret_hex())?;
tracing::info!(target: "meta-feeder::claims", uid = %signer.uid(),
path = %path.display(), "generated a new signing key");
Ok(signer)
}
Err(e) => Err(anyhow::anyhow!("read signing key {}: {e}", path.display())),
}
}
#[cfg(unix)]
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)?;
f.write_all(hex.as_bytes())?;
f.write_all(b"\n")
}
#[cfg(not(unix))]
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
std::fs::write(path, format!("{hex}\n"))
}
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key)
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
}
fn decode_hex(s: &str) -> Option<Vec<u8>> {
if s.len() % 2 != 0 {
return None;
}
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Deserialize)]
struct Vector {
name: String,
cid: String,
uid: String,
key: String,
value: Option<String>,
#[serde(rename = "valueBytesHex")]
value_bytes_hex: Option<String>,
ts: u64,
#[serde(rename = "signedBytesHex")]
signed: String,
#[serde(rename = "sigHex")]
sig: String,
}
#[derive(Deserialize)]
struct HighS {
vector: String,
#[serde(rename = "sigHex")]
sig: String,
}
#[derive(Deserialize)]
struct File {
#[serde(rename = "privateKeyHex")]
private_key_hex: String,
uid: String,
vectors: Vec<Vector>,
#[serde(rename = "highS")]
high_s: HighS,
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn load() -> File {
serde_json::from_str(include_str!("../tests/claim-vectors.json")).unwrap()
}
fn value(v: &Vector) -> ClaimValue {
match (&v.value_bytes_hex, &v.value) {
(Some(h), _) => ClaimValue::Bytes(decode_hex(h).unwrap()),
(None, Some(t)) => ClaimValue::Text(t.clone()),
(None, None) => ClaimValue::Null,
}
}
#[test]
fn golden_vectors() {
let f = load();
let signer = Signer::from_hex(&f.private_key_hex).unwrap();
assert_eq!(signer.uid(), f.uid);
for v in &f.vectors {
let c = signer.sign(&v.cid, &v.key, value(v), v.ts);
assert_eq!(c.uid, v.uid, "{}", v.name);
assert_eq!(hex(&c.signed_bytes()), v.signed, "{}: signed bytes", v.name);
assert_eq!(hex(&c.sig), v.sig, "{}: RFC 6979 signature", v.name);
c.verify().unwrap_or_else(|e| panic!("{}: {e}", v.name));
// JSON round trip keeps the claim verifiable.
let back: SignedClaim =
serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap();
assert_eq!(back, c, "{}: json", v.name);
}
let v = f
.vectors
.iter()
.find(|v| v.name == f.high_s.vector)
.unwrap();
let high = SignedClaim {
cid: v.cid.clone(),
uid: v.uid.clone(),
key: v.key.clone(),
value: value(v),
ts: v.ts,
sig: decode_hex(&f.high_s.sig).unwrap(),
};
assert!(matches!(high.verify(), Err(ClaimError::HighS)));
}
#[test]
fn tamper_is_refused() {
let s = Signer::generate();
let mut c = s.sign("bafy", "title", ClaimValue::Text("A".into()), 1);
c.value = ClaimValue::Text("B".into());
assert!(matches!(c.verify(), Err(ClaimError::Invalid)));
}
#[test]
fn key_file_is_generated_once() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("keys/signing.key");
let a = load_or_create_key_file(&p).unwrap();
let b = load_or_create_key_file(&p).unwrap();
assert_eq!(a.uid(), b.uid());
}
#[test]
fn json_shapes() {
let s = Signer::generate();
let j =
serde_json::to_value(s.sign("c", "k", ClaimValue::Bytes(vec![1, 2]), 5)).unwrap();
assert_eq!(j["value"]["b64"], "AQI=");
let j = serde_json::to_value(s.retract("c", "k", 6)).unwrap();
assert!(j["value"].is_null());
}
#[test]
fn union_keeps_first_text_and_skips_retractions_and_cipher() {
let s = Signer::generate();
let claims = vec![
s.sign("c", "title", ClaimValue::Text("A".into()), 1),
s.sign("c", "title", ClaimValue::Text("B".into()), 2),
s.retract("c", "gone", 3),
s.sign("c", "seek", ClaimValue::Bytes(vec![9]), 4),
];
let f = union_fields(&claims);
assert_eq!(f.get("title").map(String::as_str), Some("A"));
assert!(!f.contains_key("gone") && !f.contains_key("seek"));
}
}
}
+31 -7
View File
@@ -64,6 +64,10 @@ pub struct EnrichmentConfig {
pub opensubtitles_password: Option<String>, pub opensubtitles_password: Option<String>,
/// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
pub opensubtitles_languages: String, pub opensubtitles_languages: String,
/// This feeder's claim signer (`METAMESH_SIGNING_KEY` / `_FILE`). `None`
/// when no key is configured: every meta-core write then fails loudly
/// instead of publishing unsigned data.
pub signer: Option<crate::claims::Signer>,
} }
fn env_nonempty(key: &str) -> Option<String> { fn env_nonempty(key: &str) -> Option<String> {
@@ -120,6 +124,14 @@ impl EnrichmentConfig {
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
.unwrap_or_else(|| "en".to_string()), .unwrap_or_else(|| "en".to_string()),
signer: match crate::claims::load_or_create_key() {
Ok(s) => Some(s),
Err(e) => {
warn!(target: "meta-feeder::enrich", error = %e,
"no signing key; meta-core writes will fail");
None
}
},
} }
} }
} }
@@ -218,7 +230,14 @@ impl Enricher {
&self.cfg.gateway_peer, &self.cfg.gateway_peer,
); );
let body = meta_core::build_metadata_body(record, &prov); let body = meta_core::build_metadata_body(record, &prov);
meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await meta_core::put_record(&self.http, &self.cfg.meta_core_url, self.signer()?, cid, &body).await
}
fn signer(&self) -> anyhow::Result<&crate::claims::Signer> {
self.cfg
.signer
.as_ref()
.ok_or_else(|| anyhow::anyhow!("no signing key (METAMESH_SIGNING_KEY / _FILE)"))
} }
/// Run the video enrichment pipeline for one resolved CID: filename-parser /// Run the video enrichment pipeline for one resolved CID: filename-parser
@@ -240,14 +259,14 @@ impl Enricher {
Ok(()) => { Ok(()) => {
// Poll for the merged fields so tmdb gets the clean title. // Poll for the merged fields so tmdb gets the clean title.
if let Some(merged) = self if let Some(merged) = self
.poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) .poll_until_field(cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
.await .await
{ {
meta = merged; meta = merged;
} else { } else {
debug!( debug!(
target: "meta-feeder::enrich", target: "meta-feeder::enrich",
cid, "filename-parser produced no originalTitle within timeout; \ cid, "filename-parser produced no parsedTitle within timeout; \
handing tmdb the base fields" handing tmdb the base fields"
); );
} }
@@ -324,7 +343,7 @@ impl Enricher {
// Poll the primary for filename-parser's clean title, then tmdb it. // Poll the primary for filename-parser's clean title, then tmdb it.
let merged = self let merged = self
.poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) .poll_until_field(&p.cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
.await .await
.unwrap_or_else(|| p.base.clone()); .unwrap_or_else(|| p.base.clone());
@@ -355,9 +374,14 @@ impl Enricher {
} }
// PUT is merge/upsert — this layers the show-level anchor onto each // PUT is merge/upsert — this layers the show-level anchor onto each
// file's record without disturbing its per-file fields. // file's record without disturbing its per-file fields.
if let Err(e) = let res = match self.signer() {
meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await Ok(signer) => {
{ meta_core::put_record(&self.http, &self.cfg.meta_core_url, signer, &v.cid, &anchor)
.await
}
Err(e) => Err(e),
};
if let Err(e) = res {
warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
} }
} }
+208
View File
@@ -728,6 +728,159 @@ pub fn extract_codec(title: &str) -> Option<String> {
Some(norm.to_string()) Some(norm.to_string())
} }
// ---------------------------------------------------------------------------
// Release facts — `sourceMedium`, `releaseGroup`, `editions/*`, `flags/*`
// (METADATA_KEYS §3). Durable properties of a release (unlike its seeders), so
// they belong on the record; title-derived and best-effort like `quality` and
// `codec`, so absence means "couldn't tell". Each returns a closed vocabulary
// and drops what it does not recognise. The same functions normalise a
// structured value an upstream already parsed (AIOStreams' `quality`
// "BluRay REMUX", its editions "Director's Cut"), so the two paths agree.
// ---------------------------------------------------------------------------
/// Separators to spaces, the way every extractor here reads a title. Hyphens
/// are kept: `WEB-DL`, `HD-Rip` and the `-GROUP` suffix need them.
fn spaced(title: &str) -> String {
title.replace(['.', '_'], " ")
}
/// The medium the release was taken from, one of `cam`, `ts`, `tc`, `scr`,
/// `bluray-remux`, `bluray`, `webrip`, `web-dl`, `hdrip`, `hdtv`, `dvd`.
///
/// Pre-release sources are tested first: `HDCAM` beats any `HD` token, and a
/// `DVDSCR` is a screener, not a DVD. The bare `CAM`, `TS` and `TC` tokens only
/// count in capitals: as words they are a film title (*Cam*, 2018), a file
/// extension and an initialism too often.
pub fn extract_source_medium(title: &str) -> Option<&'static str> {
static CI: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\b(?:hd-?cam|cam-?rip)\b", "cam"),
(r"(?i)\b(?:telesync|hd-?ts|pdvd)\b", "ts"),
(r"(?i)\b(?:telecine|hd-?tc)\b", "tc"),
(r"(?i)\b(?:dvd-?scr|screener|scr)\b", "scr"),
(r"(?i)\b(?:bd-?remux|remux)\b", "bluray-remux"),
(r"(?i)\b(?:blu-?ray|bd-?rip|br-?rip|bdmv|bd25|bd50)\b", "bluray"),
(r"(?i)\bweb-?rip\b", "webrip"),
(r"(?i)\b(?:web-?dl|web)\b", "web-dl"),
(r"(?i)\bhd-?rip\b", "hdrip"),
(r"(?i)\b(?:hdtv|hdtvrip|uhdtv|pdtv|sdtv|tvrip|dsr)\b", "hdtv"),
(r"(?i)\b(?:dvd-?rip|dvd-?r|dvd5|dvd9|dvd)\b", "dvd"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("source re compiles"), v))
.collect()
});
static CAPS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[(r"\bCAM\b", "cam"), (r"\bTS\b", "ts"), (r"\bTC\b", "tc")]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("caps re compiles"), v))
.collect()
});
static DVD_REMUX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bdvd\S*\b.*\bremux\b|\bremux\b.*\bdvd").expect("dvd remux re"));
let t = spaced(title);
// A DVD remux is a DVD: `remux` alone names no disc.
if DVD_REMUX.is_match(&t) {
return Some("dvd");
}
CAPS.iter()
.find(|(re, _)| re.is_match(&t))
.or_else(|| CI.iter().find(|(re, _)| re.is_match(&t)))
.map(|(_, v)| *v)
}
/// The group that made the release: a leading `[Group]` (fansub style) or a
/// trailing `-GROUP` (scene style), the same two rules as meta-watch's
/// `parseVersion`. Trailing tracker tags (`[rarbg]`, `[TGx]`), a video
/// extension and a usenet `-xpost` are removed first; a candidate that is
/// really a format token (`WEB-DL` → `DL`, `x264`, `1080p`) is refused.
pub fn extract_release_group(title: &str) -> Option<String> {
static LEADING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^\s*\[([^\[\]]{2,32})\]").expect("leading group re"));
static TRAILING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"-\s*([A-Za-z0-9][A-Za-z0-9&@]{1,24})\s*$").expect("trailing group re"));
static TAIL_NOISE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)(?:\s*(?:\[[^\]]*\]|\([^)]*\))|\.(?:mkv|mp4|avi|m4v|ts|m2ts|webm|wmv|mov)|-xpost)+\s*$")
.expect("tail noise re")
});
static FORMAT: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:[xh]26[45]|hevc|avc|av1|\d{3,4}[pi]|s\d+(?:e\d+)?)$").expect("format re")
});
let not_a_format = |g: &str| {
let l = g.to_ascii_lowercase();
!(matches!(
l.as_str(),
"dl" | "rip" | "web" | "hd" | "hdr" | "dts" | "ac3" | "aac" | "remux" | "sample" | "eng" | "multi"
) || l.chars().all(|c| c.is_ascii_digit())
|| FORMAT.is_match(&l))
};
if let Some(g) = LEADING.captures(title).map(|c| c[1].trim().to_string()) {
// `[1080p]` or `[HEVC]` at the front is a tag, not a group.
if not_a_format(&g) && extract_quality(&g).is_none() {
return Some(g);
}
}
let mut t = title.trim().to_string();
loop {
let cut = TAIL_NOISE.replace(&t, "").trim_end().to_string();
if cut == t {
break;
}
t = cut;
}
TRAILING
.captures(&t)
.map(|c| c[1].to_string())
.filter(|g| not_a_format(g))
}
/// The cut of the work this release carries — `editions/<x>` members. A
/// different cut has a different runtime, so these matter beyond a label
/// (a subtitle synced to the theatrical cut drifts on the extended one).
pub fn extract_editions(title: &str) -> BTreeSet<&'static str> {
static EDITIONS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\bextended\b", "extended"),
(r"(?i)\bdirector'?s?[\s-]*cut\b", "directors-cut"),
(r"(?i)\btheatrical\b", "theatrical"),
(r"(?i)\bunrated\b", "unrated"),
(r"(?i)\buncut\b", "uncut"),
(r"(?i)\buncensored\b", "uncensored"),
(r"(?i)\bimax\b", "imax"),
(r"(?i)\bremaster(?:ed)?\b", "remastered"),
(r"(?i)\bcriterion\b", "criterion"),
(r"(?i)\banniversary\b", "anniversary"),
(r"(?i)\bcollector'?s\b", "collectors"),
(r"(?i)\bultimate[\s-]+(?:edition|cut)\b", "ultimate"),
(r"(?i)\bspecial[\s-]+edition\b", "special"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("edition re compiles"), v))
.collect()
});
let t = spaced(title);
EDITIONS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
}
/// Properties of the encode itself — `flags/<x>` members: `repack` and
/// `proper` (a fixed re-release that supersedes the first), `upscaled` (the
/// resolution is not the source's: a 2160 upscale is not 4K), `regraded`.
pub fn extract_flags(title: &str) -> BTreeSet<&'static str> {
static FLAGS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\b(?:repack\d?|rerip)\b", "repack"),
(r"(?i)\bproper\b", "proper"),
(r"(?i)\b(?:ai[\s-]*)?upscaled?\b", "upscaled"),
(r"(?i)\bregraded?\b", "regraded"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("flag re compiles"), v))
.collect()
});
let t = spaced(title);
FLAGS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
}
/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + /// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
/// `cleanSpace`, in that order. /// `cleanSpace`, in that order.
fn preclean(filename: &str) -> String { fn preclean(filename: &str) -> String {
@@ -935,6 +1088,61 @@ pub fn clean_title(raw: &str) -> String {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn source_medium_reads_real_release_names() {
let m = extract_source_medium;
assert_eq!(m("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR"), Some("bluray-remux"));
assert_eq!(m("The.Matrix.1999.1080p.BrRip.x264.YIFY.mp4"), Some("bluray"));
assert_eq!(m("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv"), Some("web-dl"));
assert_eq!(m("Show.S01E01.1080p.WEB.H264-GROUP"), Some("web-dl"));
assert_eq!(m("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), Some("webrip"));
assert_eq!(m("Movie 2023 HC HD-Rip 720p"), Some("hdrip"));
assert_eq!(m("Some Show [2x10] HDTV"), Some("hdtv"));
assert_eq!(m("Old.Film.1985.DVDRip.XviD"), Some("dvd"));
assert_eq!(m("Old.Film.1985.DVD.REMUX"), Some("dvd"));
assert_eq!(m("New.Film.2024.HDCAM.x264"), Some("cam"));
assert_eq!(m("New Film 2024 CAM"), Some("cam"));
assert_eq!(m("New.Film.2024.DVDSCR.x264"), Some("scr"));
assert_eq!(m("New.Film.2024.TELESYNC"), Some("ts"));
// AIOStreams' structured vocabulary goes through the same door.
assert_eq!(m("BluRay REMUX"), Some("bluray-remux"));
assert_eq!(m("WEB-DL"), Some("web-dl"));
assert_eq!(m("TC"), Some("tc"));
// The film "Cam" and a lower-case "ts" are not sources.
assert_eq!(m("Cam (2018) 1080p"), None);
assert_eq!(m("its a ts file"), None);
assert_eq!(m("Movie 2019 1080p x264"), None);
}
#[test]
fn release_group_is_the_leading_bracket_or_trailing_dash() {
let g = |t: &str| extract_release_group(t);
assert_eq!(g("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)").as_deref(), Some("yolerejiju"));
assert_eq!(g("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv").as_deref(), Some("G66"));
assert_eq!(g("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR").as_deref(), Some("FraMeSToR"));
assert_eq!(g("Oppenheimer.2023.1080p.BluRay.DD5.1.x264-GalaxyRG.mkv").as_deref(), Some("GalaxyRG"));
assert_eq!(g("Movie.2020.1080p.WEB-DL.x264-NTb[rarbg]").as_deref(), Some("NTb"));
assert_eq!(g("Breaking.Bad.S02E04.Down.1080p.BluRay.x264-OFT.mkv-xpost").as_deref(), Some("OFT"));
assert_eq!(g("Movie 2020 1080p WEB-DL [TGx]").as_deref(), None, "WEB-DL is not a group called DL");
assert_eq!(g("[1080p] Movie - 2020").as_deref(), None);
assert_eq!(g("Movie.2020.1080p.x264").as_deref(), None);
}
#[test]
fn editions_and_flags() {
let e = |t: &str| extract_editions(t).into_iter().collect::<Vec<_>>();
assert_eq!(e("Aliens.1986.Directors.Cut.1080p.BluRay"), ["directors-cut"]);
assert_eq!(e("LOTR.Fellowship.2001.EXTENDED.REMASTERED.2160p"), ["extended", "remastered"]);
assert_eq!(e("Director's Cut"), ["directors-cut"]);
assert_eq!(e("Ultimate Edition"), ["ultimate"]);
assert_eq!(e("Movie.2019.1080p.WEB-DL"), Vec::<&str>::new());
let f = |t: &str| extract_flags(t).into_iter().collect::<Vec<_>>();
assert_eq!(f("Game of Thrones S01E03 Lord Snow REPACK 1080p"), ["repack"]);
assert_eq!(f("Show.S01E01.PROPER.720p.HDTV"), ["proper"]);
assert_eq!(f("Movie 1999 2160p AI Upscaled Regrade"), ["regraded", "upscaled"]);
assert_eq!(f("The Proper Way 2020"), ["proper"], "a title word is indistinguishable — accepted noise");
}
fn se(title: &str) -> (Option<String>, Option<String>) { fn se(title: &str) -> (Option<String>, Option<String>) {
let r = extract_season_episode(title); let r = extract_season_episode(title);
(r.season, r.episode) (r.season, r.episode)
+9 -1
View File
@@ -34,6 +34,8 @@ pub mod beacon;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod budget; pub mod budget;
#[cfg(any(feature = "feeder", feature = "claims"))]
pub mod claims;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod cache; pub mod cache;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
@@ -44,7 +46,7 @@ pub mod config;
pub mod domain; pub mod domain;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod enrich; pub mod enrich;
#[cfg(feature = "feeder")] #[cfg(feature = "filename")]
pub mod filename_meta; pub mod filename_meta;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod hash; pub mod hash;
@@ -52,6 +54,8 @@ pub mod hash;
pub mod lang; pub mod lang;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod licence; pub mod licence;
/// Search-wire-transient `live/*` fields — always available, no dependencies.
pub mod live;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub mod meta_core; pub mod meta_core;
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
@@ -90,5 +94,9 @@ pub use serve::{
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
RedeemsResponse, RedeemsResponse,
}; };
#[cfg(any(feature = "feeder", feature = "claims"))]
pub use claims::{ClaimValue, SignedClaim};
#[cfg(feature = "claims")]
pub use claims::{load_or_create_key, ClaimError, Signer};
#[cfg(feature = "feeder")] #[cfg(feature = "feeder")]
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
+79
View File
@@ -0,0 +1,79 @@
//! Search-wire-transient fields: facts a feeder sees at query time that are
//! true only *now* — a torrent's seeders, its leechers — and so must never
//! reach the replicated `/file/{cid}` hash (METADATA_KEYS rule #4).
//!
//! They travel under one reserved prefix, [`LIVE_PREFIX`], so every layer that
//! persists a record strips the whole family without knowing each key:
//!
//! ```text
//! live/seeders = "42" int-string
//! live/leechers = "3" int-string
//! live/at = "1791726110" unix seconds the feeder observed them
//! ```
//!
//! A record without them means "unknown", never "zero": a replayed or
//! persisted record has none, and that is the honest answer.
//!
//! The feeder SDK strips them in [`crate::meta_core::build_metadata_body`]
//! (every feeder self-publish); meta-gateway and meta-search strip them on
//! their own write paths (see METADATA_KEYS "Search-wire-transient fields").
use std::collections::BTreeMap;
/// The reserved prefix of the transient family.
pub const LIVE_PREFIX: &str = "live/";
/// Swarm seeders, `int-string`.
pub const LIVE_SEEDERS: &str = "live/seeders";
/// Swarm leechers, `int-string`.
pub const LIVE_LEECHERS: &str = "live/leechers";
/// When the feeder observed the `live/*` values, unix seconds.
pub const LIVE_AT: &str = "live/at";
/// The bare swarm counters some feeders wrote before the `live/*` family
/// existed (tribler). Stripped with it: rule #4 bars them in any spelling.
pub const LEGACY_SWARM_KEYS: [&str; 3] = ["seeders", "leechers", "peers"];
/// Whether a field is search-wire-transient and must not be persisted.
pub fn is_transient(key: &str) -> bool {
key.starts_with(LIVE_PREFIX) || LEGACY_SWARM_KEYS.contains(&key)
}
/// Remove every transient field. Call on the copy about to be persisted, never
/// on the record still bound for the search wire.
pub fn strip_transient(fields: &mut BTreeMap<String, String>) {
fields.retain(|k, _| !is_transient(k));
}
/// The `live/at` value for a feeder observing counters right now.
pub fn now_secs() -> String {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strips_the_family_and_the_legacy_counters_only() {
let mut f: BTreeMap<String, String> = [
("live/seeders", "42"),
("live/at", "1"),
("seeders", "7"),
("peers", "9"),
("fileName", "x.mkv"),
("livestream", "true"),
("source/live", "true"),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
strip_transient(&mut f);
let keys: Vec<&str> = f.keys().map(String::as_str).collect();
assert_eq!(keys, ["fileName", "livestream", "source/live"]);
}
}
+119 -21
View File
@@ -12,10 +12,12 @@
//! written by the feeder is indistinguishable from one the dispatcher wrote — //! written by the feeder is indistinguishable from one the dispatcher wrote —
//! the meta-share ingester, meta-watch, and ranking all behave the same. //! the meta-share ingester, meta-watch, and ranking all behave the same.
//! //!
//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core //! Since meta-core v2.0.0 every write is a batch of **signed claims** to
//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the //! `POST /api/claims` (spec `docs/implementation/metacore-claims-model.md`):
//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}` //! each field becomes one MMUR2 vote signed by this feeder's own key. Votes from
//! merges accumulate on the same `/file/{cid}` hash. //! different writers never overwrite each other — meta-core merges them — so the
//! feeder's base write and the enrichment plugins' later claims accumulate on
//! the same cid.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@@ -23,6 +25,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use serde::Deserialize; use serde::Deserialize;
use tracing::warn; use tracing::warn;
use crate::claims::{now_ms, ClaimValue, SignedClaim, Signer};
use crate::plugin::{HashKind, HashOutcome}; use crate::plugin::{HashKind, HashOutcome};
use crate::types::{DiscoveryRecord, Hash}; use crate::types::{DiscoveryRecord, Hash};
@@ -80,6 +83,11 @@ pub fn build_metadata_body(
) -> BTreeMap<String, String> { ) -> BTreeMap<String, String> {
let mut body: BTreeMap<String, String> = BTreeMap::new(); let mut body: BTreeMap<String, String> = BTreeMap::new();
for (k, v) in &record.fields { for (k, v) in &record.fields {
// `live/*` (and the legacy bare swarm counters) are true only at query
// time; a self-publishing feeder must not freeze them into the hash.
if crate::live::is_transient(k) {
continue;
}
if k.starts_with("cid_") && !v.is_empty() { if k.starts_with("cid_") && !v.is_empty() {
body.insert(format!("cids/{v}"), "true".to_string()); body.insert(format!("cids/{v}"), "true".to_string());
} else { } else {
@@ -90,22 +98,76 @@ pub fn build_metadata_body(
body body
} }
/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into /// One per-claim verdict from `POST /api/claims`.
/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`). #[derive(Debug, Deserialize)]
pub async fn put_record( pub struct ClaimStatus {
pub status: String,
#[serde(default)]
pub reason: Option<String>,
}
/// `POST /api/claims` — submit already-signed claims (e.g. relayed verbatim).
/// Fails on a non-2xx answer or when any claim is refused as `bad_signature` /
/// `bad_request`; `stale` and `unchanged` are normal outcomes.
pub async fn post_claims(
http: &reqwest::Client, http: &reqwest::Client,
base_url: &str, base_url: &str,
cid: &str, claims: &[SignedClaim],
body: &BTreeMap<String, String>, ) -> anyhow::Result<Vec<ClaimStatus>> {
) -> anyhow::Result<()> { if claims.is_empty() {
let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid); return Ok(Vec::new());
let resp = http.put(&url).json(body).send().await?; }
let url = format!("{}/api/claims", base_url.trim_end_matches('/'));
let resp = http.post(&url).json(claims).send().await?;
let status = resp.status(); let status = resp.status();
if !status.is_success() { if !status.is_success() {
let text = resp.text().await.unwrap_or_default(); let text = resp.text().await.unwrap_or_default();
anyhow::bail!("meta-core PUT {url} returned {status}: {text}"); anyhow::bail!("meta-core POST {url} returned {status}: {text}");
} }
Ok(()) let verdicts: Vec<ClaimStatus> = resp.json().await?;
if let Some((i, bad)) = verdicts
.iter()
.enumerate()
.find(|(_, v)| v.status == "bad_signature" || v.status == "bad_request")
{
anyhow::bail!(
"meta-core refused claim {} ({}): {} {}",
i,
claims.get(i).map(|c| c.key.as_str()).unwrap_or("?"),
bad.status,
bad.reason.as_deref().unwrap_or("")
);
}
Ok(verdicts)
}
/// Sign every field of `body` as a text claim on `cid` (all at "now") and
/// submit them. The claims-model replacement for the v1 `PUT /api/metadata`.
pub async fn put_record(
http: &reqwest::Client,
base_url: &str,
signer: &Signer,
cid: &str,
body: &BTreeMap<String, String>,
) -> anyhow::Result<()> {
let claims = signer.sign_fields(cid, body, now_ms());
post_claims(http, base_url, &claims).await.map(|_| ())
}
/// Sign retractions of `keys` on `cid` (this signer's own votes only).
pub async fn retract_keys(
http: &reqwest::Client,
base_url: &str,
signer: &Signer,
cid: &str,
keys: &[String],
) -> anyhow::Result<()> {
let ts = now_ms();
let claims: Vec<SignedClaim> = keys
.iter()
.map(|k| signer.sign(cid, k, ClaimValue::Null, ts))
.collect();
post_claims(http, base_url, &claims).await.map(|_| ())
} }
/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`). /// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`).
@@ -157,27 +219,49 @@ pub struct FeederStore {
http: reqwest::Client, http: reqwest::Client,
meta_core_url: String, meta_core_url: String,
gateway_peer: String, gateway_peer: String,
signer: Signer,
} }
impl FeederStore { impl FeederStore {
pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self { pub fn new(
http: reqwest::Client,
meta_core_url: String,
gateway_peer: String,
signer: Signer,
) -> Self {
FeederStore { FeederStore {
http, http,
meta_core_url: meta_core_url.trim_end_matches('/').to_string(), meta_core_url: meta_core_url.trim_end_matches('/').to_string(),
gateway_peer, gateway_peer,
signer,
} }
} }
/// Build from the feeder env. `None` when `META_CORE_URL` is unset (the /// Build from the feeder env. `None` when `META_CORE_URL` is unset, or when
/// feeder then falls back to the legacy "return outcomes, gateway stores" /// no signing key is configured (`METAMESH_SIGNING_KEY` /
/// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`. /// `METAMESH_SIGNING_KEY_FILE`) — the feeder then falls back to the legacy
/// "return outcomes, gateway stores" path. `gateway_peer` ←
/// `META_GATEWAY_PEER_ID` / `HOSTNAME`.
pub fn from_env() -> Option<Self> { pub fn from_env() -> Option<Self> {
let meta_core_url = env_nonempty("META_CORE_URL")?; let meta_core_url = env_nonempty("META_CORE_URL")?;
let signer = match crate::claims::load_or_create_key() {
Ok(s) => s,
Err(e) => {
warn!(target: "meta-feeder::store", error = %e,
"META_CORE_URL set but no signing key; not self-publishing");
return None;
}
};
let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID") let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID")
.or_else(|| env_nonempty("HOSTNAME")) .or_else(|| env_nonempty("HOSTNAME"))
.unwrap_or_else(|| "gateway-feeder".to_string()); .unwrap_or_else(|| "gateway-feeder".to_string());
let http = crate::common::build_http_client(30, "meta-feeder-store", None); let http = crate::common::build_http_client(30, "meta-feeder-store", None);
Some(Self::new(http, meta_core_url, gateway_peer)) Some(Self::new(http, meta_core_url, gateway_peer, signer))
}
/// The uid this store signs as.
pub fn signer(&self) -> &Signer {
&self.signer
} }
pub fn meta_core_url(&self) -> &str { pub fn meta_core_url(&self) -> &str {
@@ -257,7 +341,8 @@ impl FeederStore {
} }
} }
/// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance. /// Metadata-only self-publish: the record's fields (+ provenance) as signed
/// claims to `POST /api/claims`.
async fn store_metadata_only( async fn store_metadata_only(
&self, &self,
upstream_id: &str, upstream_id: &str,
@@ -267,7 +352,7 @@ impl FeederStore {
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer); let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer);
let body = build_metadata_body(record, &prov); let body = build_metadata_body(record, &prov);
put_record(&self.http, &self.meta_core_url, cid, &body).await put_record(&self.http, &self.meta_core_url, &self.signer, cid, &body).await
} }
} }
@@ -300,6 +385,7 @@ mod tests {
("title".into(), "Big Buck Bunny".into()), ("title".into(), "Big Buck Bunny".into()),
("cid_btih_v1_file".into(), "bafkPERFILE".into()), ("cid_btih_v1_file".into(), "bafkPERFILE".into()),
]), ]),
claims: Vec::new(),
} }
} }
@@ -332,4 +418,16 @@ mod tests {
); );
assert!(body.contains_key("provenance")); assert!(body.contains_key("provenance"));
} }
/// A self-publish never freezes swarm counters, in either spelling.
#[test]
fn body_drops_transient_fields() {
let mut r = rec();
for (k, v) in [("live/seeders", "42"), ("live/at", "1"), ("seeders", "7"), ("leechers", "2")] {
r.fields.insert(k.into(), v.into());
}
let body = build_metadata_body(&r, &Provenance::now("tribler", "abc", "peer"));
assert!(body.keys().all(|k| !crate::live::is_transient(k)), "{body:?}");
assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny"));
}
} }
+4
View File
@@ -100,6 +100,10 @@ pub enum GatewaySearchEvent {
record_id: String, record_id: String,
set: BTreeMap<String, String>, set: BTreeMap<String, String>,
remove: Vec<String>, remove: Vec<String>,
/// Signed form of the patch: `set` as text claims, `remove` as
/// retractions. Empty from a feeder (the gateway core signs).
#[serde(default)]
claims: Vec<crate::claims::SignedClaim>,
}, },
/// Retract a previously-emitted `Base` record entirely. /// Retract a previously-emitted `Base` record entirely.
Drop { record_id: String }, Drop { record_id: String },
+2
View File
@@ -799,6 +799,7 @@ mod tests {
upstream_id: "stub".into(), upstream_id: "stub".into(),
record_id: "1".into(), record_id: "1".into(),
fields: BTreeMap::new(), fields: BTreeMap::new(),
claims: Vec::new(),
}; };
let events = vec![ let events = vec![
GatewaySearchEvent::Base(rec), GatewaySearchEvent::Base(rec),
@@ -806,6 +807,7 @@ mod tests {
record_id: "1".into(), record_id: "1".into(),
set: BTreeMap::from([("title".to_string(), "X".to_string())]), set: BTreeMap::from([("title".to_string(), "X".to_string())]),
remove: vec![], remove: vec![],
claims: vec![],
}, },
GatewaySearchEvent::Done, GatewaySearchEvent::Done,
]; ];
+19
View File
@@ -156,6 +156,25 @@ impl RemoteTransport {
} }
} }
/// `PUT /manifests/:cid`: hand a manifest-consuming plugin (contract 2) the
/// manifest a pointer resolved to, after it answered `428` +
/// [`super::dto::HDR_NEEDS_MANIFEST`].
pub async fn push_manifest(&self, cid: &str, manifest: bytes::Bytes) -> Result<(), ApiError> {
let r = self
.http
.put(self.url(&format!("/manifests/{cid}")))
.body(manifest)
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
if r.status().is_success() {
Ok(())
} else {
Err(error_from(r).await)
}
}
// ---- typed helpers for extra routes ------------------------------------ // ---- typed helpers for extra routes ------------------------------------
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> { pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
+31 -2
View File
@@ -9,8 +9,37 @@ use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
/// Bumped on any breaking change to the common routes below. The hull refuses /// Bumped on any breaking change to the common routes below. The hull refuses
/// a plugin whose `/manifest` reports a different major contract. /// a plugin whose `/manifest` reports a contract outside
pub const CONTRACT_VERSION: u32 = 1; /// [`MIN_CONTRACT`]`..=`[`CONTRACT_VERSION`].
///
/// - **2**: a plugin that consumes a manifest (nzb) no longer pulls it from the
/// hull. It answers `/raw` (and its verify route) with `428` +
/// [`HDR_NEEDS_MANIFEST`] until the hull has pushed one with
/// `PUT /manifests/:cid`.
pub const CONTRACT_VERSION: u32 = 2;
/// The oldest contract the hull still drives (contract-1 nzb plugins pull their
/// manifest from `/internal/nzb/manifest/:cid`, kept for one release).
pub const MIN_CONTRACT: u32 = 1;
/// `428` marker: this plugin needs the cid's manifest pushed before it can
/// answer. The hull resolves the pointer, `PUT`s `/manifests/:cid`, and retries.
pub const HDR_NEEDS_MANIFEST: &str = "x-metamesh-needs-manifest";
/// Why a pointer could not be resolved, on a non-2xx `/ipfs-tier/resolve`
/// answer: [`RESOLVE_NO_GATEWAY`], [`RESOLVE_UNCLAIMED`], [`RESOLVE_QUOTA`],
/// [`RESOLVE_NOT_FOUND`] or [`RESOLVE_UPSTREAM`].
pub const HDR_RESOLVE_ERROR: &str = "x-metamesh-resolve-error";
/// No gateway is known at all yet (discovery is eventual): retry.
pub const RESOLVE_NO_GATEWAY: &str = "no-gateway";
/// Gateways are known, but none claims this pointer's key.
pub const RESOLVE_UNCLAIMED: &str = "unclaimed";
/// The provider's quota is spent: retry after `Retry-After`.
pub const RESOLVE_QUOTA: &str = "quota";
/// Every claiming gateway answered not found (or nothing is stored and the
/// pointer cannot be redeemed).
pub const RESOLVE_NOT_FOUND: &str = "not-found";
/// A gateway failed, or answered bytes that did not verify.
pub const RESOLVE_UPSTREAM: &str = "upstream";
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request. /// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
pub const HDR_LANE: &str = "x-metamesh-lane"; pub const HDR_LANE: &str = "x-metamesh-lane";
+4 -3
View File
@@ -39,10 +39,11 @@ use super::dto::Event;
/// Default hull callback base, matching the dev and store compose service name. /// Default hull callback base, matching the dev and store compose service name.
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001"; pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
/// Where the hull got a manifest: the record's `manifest` pointer, or a /// Where a pointer's bytes came from: `store` (already held here, named by the
/// gateway redeem. /// record's pointer field), `redeem` (a gateway), or `record` (contract-1
/// callback naming for `store`).
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source"; pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
/// The content cid of a `record`-sourced manifest. /// The content cid the pointer resolved to.
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid"; pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
/// `.nzb` bytes as the hull resolved them. /// `.nzb` bytes as the hull resolved them.
+10
View File
@@ -17,6 +17,7 @@
//! |---|---| //! |---|---|
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` | //! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) | //! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
//! | `GET resolve/:cid?hint=` → bytes | a [pointer](super::pointer)'s bytes: the stored copy `hint` names, else a gateway redeem (see below) |
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` | //! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` | //! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` | //! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
@@ -40,6 +41,15 @@
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for //! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
//! a display name. //! a display name.
//! **Resolving a pointer** (`GET resolve/:cid`) is the one route that reaches
//! outside the swarm: a pointer is answered only by a gateway holding the key,
//! over plain HTTP at the gateway's base URL. Gateways come from config (the
//! local one, pinned) and from the swarm directory; the fastest one whose redeem
//! claim covers the pointer is asked first. `2xx` carries the bytes plus
//! [`HDR_MANIFEST_SOURCE`](super::hull::HDR_MANIFEST_SOURCE) (`store` | `redeem`)
//! and [`HDR_MANIFEST_CID`](super::hull::HDR_MANIFEST_CID); a failure carries
//! [`HDR_RESOLVE_ERROR`](super::dto::HDR_RESOLVE_ERROR).
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)] #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
+3 -1
View File
@@ -22,6 +22,7 @@
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers. //! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade). //! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings. //! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
//! - [`pointer`]: which cids are pointers (resolved before their bytes exist).
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`. //! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
//! - `testkit` (feature `testkit`): conformance checks for any implementation. //! - `testkit` (feature `testkit`): conformance checks for any implementation.
//! //!
@@ -42,6 +43,7 @@ pub mod ipfs;
pub mod magic; pub mod magic;
pub mod nzb; pub mod nzb;
pub mod plugin; pub mod plugin;
pub mod pointer;
pub mod range; pub mod range;
pub mod serve; pub mod serve;
pub mod torrent; pub mod torrent;
@@ -52,7 +54,7 @@ pub use client::RemoteTransport;
pub use config::ConfigPlane; pub use config::ConfigPlane;
pub use dto::{ pub use dto::{
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport, Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
ReconcileRequest, CONTRACT_VERSION, ReconcileRequest, CONTRACT_VERSION, MIN_CONTRACT,
}; };
pub use error::ApiError; pub use error::ApiError;
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane}; pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
+248
View File
@@ -0,0 +1,248 @@
//! **Pointer cids** — cids that name something to fetch rather than hash it.
//!
//! A pointer's multihash is `identity`: the cid *contains* the address (an
//! indexer host + release id, a provider source + file id), not a digest of the
//! bytes. Bitswap cannot answer one, because there is no hash to verify against.
//! It is resolved instead: by a manifest this peer already stored (the record's
//! pointer field names its content cid), or by a **redeem** at a gateway whose
//! feeder holds the credential for `key`.
//!
//! This module is the single place that knows which codecs are pointers, so the
//! hull ("resolve it first") and the ipfs plugin ("here is how") agree.
//!
//! | codec | what it points at | redeem key | record field |
//! |---|---|---|---|
//! | `0x1005` nzb-release | a `.nzb` at an indexer | indexer host | `manifest` |
//! | `0x1003` nzb-posting | a self-scanned `.nzb` | — (record only) | `manifest` |
//! | `0x100A` provider-file | a file a provider holds | provider source | `file` |
use anyhow::{Context, Result};
use super::cid::{parse_record_cid, read_uvarint};
use super::nzb::{
decode_nzb_release_cid, normalize_indexer_host, NZB_POSTING_CODEC, NZB_RELEASE_CODEC,
};
/// MetaMesh-private multicodec for a provider-held file (`docs/cid-formats.md` §8).
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
/// Redeem-claim codec names, as gateways advertise them in
/// `GET /api/gateway/plugins` (`redeems[].codec`).
pub const REDEEM_NZB_RELEASE: &str = "nzb-release";
pub const REDEEM_PROVIDER_FILE: &str = "provider-file";
/// Record fields a redeem merges onto the pointer's record: the content cid of
/// what it resolved to.
pub const FIELD_MANIFEST: &str = "manifest";
pub const FIELD_FILE: &str = "file";
/// Identity multihash code: the digest bytes are the payload verbatim.
const IDENTITY_MULTIHASH: u64 = 0x00;
/// A decoded pointer.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Pointer {
/// The redeem-claim codec, or `None` when no gateway can redeem it (a
/// self-scanned posting resolves from the record alone).
pub redeem: Option<RedeemKey>,
/// The record field naming the resolved content cid.
pub field: &'static str,
/// What the resolved bytes are: a manifest a byte plugin consumes, or the
/// file itself.
pub kind: PointerKind,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RedeemKey {
/// [`REDEEM_NZB_RELEASE`] or [`REDEEM_PROVIDER_FILE`].
pub codec: &'static str,
/// The indexer host / provider source a gateway's claim must cover.
pub key: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PointerKind {
/// A `.nzb`: handed to the nzb plugin, which fetches the bytes.
NzbManifest,
/// The file itself (a subtitle): served as is.
File,
}
/// Decode `cid` as a pointer.
///
/// - `None` — not a pointer: an ordinary content cid, or unparseable.
/// - `Some(Err(_))` — the codec says pointer but the payload is malformed; the
/// caller answers `400`.
pub fn decode(cid: &str) -> Option<Result<Pointer>> {
match codec_of(cid)? {
NZB_RELEASE_CODEC => Some(decode_nzb_release(cid)),
NZB_POSTING_CODEC => Some(Ok(Pointer {
redeem: None,
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})),
PROVIDER_FILE_CODEC => Some(decode_provider_file(cid).map(|(source, _id)| Pointer {
redeem: Some(RedeemKey { codec: REDEEM_PROVIDER_FILE, key: source }),
field: FIELD_FILE,
kind: PointerKind::File,
})),
_ => None,
}
}
/// Is `cid` a pointer (well-formed or not)?
pub fn is_pointer(cid: &str) -> bool {
decode(cid).is_some()
}
fn decode_nzb_release(cid: &str) -> Result<Pointer> {
let mscid = parse_record_cid(cid)?;
let locator = decode_nzb_release_cid(&mscid)?;
Ok(Pointer {
redeem: Some(RedeemKey {
codec: REDEEM_NZB_RELEASE,
key: normalize_indexer_host(locator.authority()),
}),
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})
}
/// Decode a `provider-file` cid into `(source, id)`. Errors on a different
/// codec or a malformed payload. Works on the raw multibase string because a
/// long provider id can exceed the 64-byte multihash a typed cid parse allows.
pub fn decode_provider_file(cid: &str) -> Result<(String, String)> {
let bytes = raw_cid_bytes(cid).context("provider-file: not a base32 CIDv1")?;
let mut at = 0usize;
let next = |at: &mut usize, what: &str| -> Result<u64> {
let (v, n) = read_uvarint(&bytes[*at..])
.with_context(|| format!("provider-file: {what} varint"))?;
*at += n;
Ok(v)
};
anyhow::ensure!(next(&mut at, "version")? == 1, "provider-file: not a CIDv1");
anyhow::ensure!(next(&mut at, "codec")? == PROVIDER_FILE_CODEC, "not a provider-file cid");
let mh = next(&mut at, "multihash-code")?;
anyhow::ensure!(
mh == IDENTITY_MULTIHASH,
"provider-file locator must use the identity multihash (0x00), got 0x{mh:x}"
);
let len = next(&mut at, "digest-length")?;
anyhow::ensure!(
(bytes.len() - at) as u64 == len,
"provider-file digest length {len} != {} trailing byte(s)",
bytes.len() - at
);
let source_len = next(&mut at, "source-length")?;
let rest = &bytes[at..];
let source_len = usize::try_from(source_len)
.ok()
.filter(|n| *n <= rest.len())
.context("provider-file: source length overruns the digest")?;
let (source, id) = rest.split_at(source_len);
let source = std::str::from_utf8(source).context("provider-file source is not utf-8")?;
let id = std::str::from_utf8(id).context("provider-file id is not utf-8")?;
anyhow::ensure!(
!source.is_empty() && !id.is_empty(),
"provider-file locator needs both a source and an id"
);
Ok((source.to_string(), id.to_string()))
}
/// The codec of a base32 CIDv1 string, without the 64-byte multihash limit.
fn codec_of(cid: &str) -> Option<u64> {
let bytes = raw_cid_bytes(cid)?;
let (version, n) = read_uvarint(&bytes).ok()?;
if version != 1 {
return None;
}
read_uvarint(&bytes[n..]).ok().map(|(codec, _)| codec)
}
/// Decode a `b`-prefixed lowercase base32 (RFC 4648, no padding) CID string.
fn raw_cid_bytes(cid: &str) -> Option<Vec<u8>> {
let body = cid.strip_prefix('b')?;
let mut out = Vec::with_capacity(body.len() * 5 / 8);
let (mut acc, mut bits) = (0u32, 0u32);
for c in body.bytes() {
let v = match c {
b'a'..=b'z' => c - b'a',
b'2'..=b'7' => c - b'2' + 26,
_ => return None,
} as u32;
acc = (acc << 5) | v;
bits += 5;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// The Ragna Crimson release from watch.nsl.sh (2026-10-04).
const NZBGEEK: &str = "bagcsaabfcrqxa2jonz5gez3fmvvs42lomzxs6ylqngxuch22nhfhl4v4vmyhiegah2ka";
fn b32(bytes: &[u8]) -> String {
const A: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
let (mut acc, mut bits, mut s) = (0u32, 0u32, String::from("b"));
for &b in bytes {
acc = (acc << 8) | b as u32;
bits += 8;
while bits >= 5 {
bits -= 5;
s.push(A[((acc >> bits) & 31) as usize] as char);
}
acc &= (1 << bits) - 1;
}
if bits > 0 {
s.push(A[((acc << (5 - bits)) & 31) as usize] as char);
}
s
}
fn provider_file(source: &str, id: &str) -> String {
let mut digest = vec![source.len() as u8];
digest.extend_from_slice(source.as_bytes());
digest.extend_from_slice(id.as_bytes());
// version 1, codec 0x100A (varint 0x8a 0x20), identity, length
let mut wire = vec![0x01, 0x8a, 0x20, 0x00, digest.len() as u8];
wire.extend_from_slice(&digest);
b32(&wire)
}
#[test]
fn an_nzb_release_is_a_pointer_keyed_by_its_indexer_host() {
let p = decode(NZBGEEK).unwrap().unwrap();
assert_eq!(
p.redeem,
Some(RedeemKey { codec: REDEEM_NZB_RELEASE, key: "api.nzbgeek.info".into() })
);
assert_eq!(p.field, FIELD_MANIFEST);
assert_eq!(p.kind, PointerKind::NzbManifest);
}
#[test]
fn a_provider_file_is_a_pointer_keyed_by_its_source() {
let cid = provider_file("opensubtitles", "file:7061834");
assert_eq!(
decode_provider_file(&cid).unwrap(),
("opensubtitles".into(), "file:7061834".into())
);
let p = decode(&cid).unwrap().unwrap();
assert_eq!(p.redeem.unwrap().key, "opensubtitles");
assert_eq!(p.field, FIELD_FILE);
assert_eq!(p.kind, PointerKind::File);
}
#[test]
fn a_content_cid_is_not_a_pointer() {
assert!(!is_pointer("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy"));
assert!(!is_pointer("not a cid"));
}
}
+8
View File
@@ -49,6 +49,13 @@ pub struct DiscoveryRecord {
pub upstream_id: String, pub upstream_id: String,
pub record_id: String, pub record_id: String,
pub fields: BTreeMap<String, String>, pub fields: BTreeMap<String, String>,
/// The record's MMUR2 claims (spec §10). Empty from a feeder that returns
/// plain `fields` (the gateway core signs those as itself); filled by a
/// self-publishing feeder or by the gateway before the record goes on the
/// wire. `#[serde(default)]` keeps older feeders' JSON `/query` answers
/// decoding.
#[serde(default)] // never skip_serializing_if: bincode is positional
pub claims: Vec<crate::claims::SignedClaim>,
} }
/// In-process error returned by `FeederPlugin` methods. /// In-process error returned by `FeederPlugin` methods.
@@ -102,6 +109,7 @@ mod tests {
("title".to_string(), "AlphaFold".to_string()), ("title".to_string(), "AlphaFold".to_string()),
("year".to_string(), "2021".to_string()), ("year".to_string(), "2021".to_string()),
]), ]),
claims: Vec::new(),
}; };
let j = serde_json::to_string(&r).expect("serialize"); let j = serde_json::to_string(&r).expect("serialize");
let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize"); let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize");
+93
View File
@@ -0,0 +1,93 @@
{
"_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
"privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"vectors": [
{
"name": "text",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "title",
"value": "The Matrix",
"ts": 1793001234567,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087",
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8"
},
{
"name": "empty-text",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "title",
"value": "",
"ts": 1,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001",
"sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642"
},
{
"name": "unicode",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "titles/jpn/攻殻機動隊",
"value": "true",
"ts": 1793001234567,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087",
"sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207"
},
{
"name": "keyset-member",
"cid": "bagacbabaec7v3",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "cids/bafkreih5kapbjzq",
"value": "true",
"ts": 23,
"signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517",
"sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761"
},
{
"name": "bytes",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "seek",
"value": null,
"valueBytesHex": "00ff10ab",
"ts": 24,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818",
"sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4"
},
{
"name": "retraction",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "genres/Drama",
"value": null,
"ts": 255,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff",
"sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac"
},
{
"name": "long-value",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "plot/eng",
"value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ts": 65536,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000",
"sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2"
},
{
"name": "ts-64bit",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "year",
"value": "1999",
"ts": 1099511627776,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000",
"sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d"
}
],
"highS": {
"note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it",
"vector": "text",
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899"
}
}