Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f804d3adb1 | ||
|
|
01cc4e15a5 | ||
|
|
38ee6d8e41 | ||
|
|
612f2aed66 | ||
|
|
027917c8cf |
+21
-3
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "meta-feeder-sdk"
|
name = "meta-feeder-sdk"
|
||||||
version = "1.6.0"
|
version = "1.10.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
rust-version = "1.85"
|
rust-version = "1.85"
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
@@ -37,11 +37,15 @@ default = ["feeder"]
|
|||||||
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
|
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
|
||||||
# so existing consumers build exactly what they always did.
|
# so existing consumers build exactly what they always did.
|
||||||
feeder = [
|
feeder = [
|
||||||
"beacon",
|
"beacon", "filename", "claims",
|
||||||
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
|
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
|
||||||
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
|
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
|
||||||
"dep:regex", "dep:redb",
|
"dep:redb",
|
||||||
]
|
]
|
||||||
|
# Release-name parsing (`src/filename_meta.rs`: clean_title, season/episode,
|
||||||
|
# quality, codec). Needs only `regex`, so a client (meta-watch) can derive a
|
||||||
|
# display name from a raw release string without the feeder machinery.
|
||||||
|
filename = ["dep:regex"]
|
||||||
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
|
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
|
||||||
# so a meta-* service takes the SDK with `default-features = false,
|
# so a meta-* service takes the SDK with `default-features = false,
|
||||||
# features = ["beacon"]` and pulls in none of the feeder machinery.
|
# features = ["beacon"]` and pulls in none of the feeder machinery.
|
||||||
@@ -52,6 +56,15 @@ beacon = ["dep:libc"]
|
|||||||
transport = ["feeder", "dep:cid", "dep:tokio-util"]
|
transport = ["feeder", "dep:cid", "dep:tokio-util"]
|
||||||
# Conformance checks any transport implementation must pass.
|
# Conformance checks any transport implementation must pass.
|
||||||
testkit = ["transport"]
|
testkit = ["transport"]
|
||||||
|
# MMUR2 signed claims (`src/claims.rs`): the `SignedClaim` type, its canonical
|
||||||
|
# bytes, signing (k256, compact low-S), verification and the writer key
|
||||||
|
# (`METAMESH_SIGNING_KEY` / `_FILE`). `feeder` enables it because the gateway
|
||||||
|
# wire (`DiscoveryRecord`, `EnrichPatch`) carries claims; a meta-* service that
|
||||||
|
# only needs claims takes `default-features = false, features = ["claims"]`.
|
||||||
|
claims = [
|
||||||
|
"dep:k256", "dep:bs58", "dep:sha2", "dep:base64", "dep:rand_core",
|
||||||
|
"dep:anyhow", "dep:thiserror",
|
||||||
|
]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# Async runtime + HTTP server (the serve_feeders harness)
|
# Async runtime + HTTP server (the serve_feeders harness)
|
||||||
@@ -76,6 +89,11 @@ bytes = { version = "1", optional = true }
|
|||||||
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
|
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
|
||||||
regex = { version = "1", optional = true }
|
regex = { version = "1", optional = true }
|
||||||
|
|
||||||
|
# MMUR2 claims (the `claims` feature).
|
||||||
|
k256 = { version = "0.13", features = ["ecdsa", "std"], optional = true }
|
||||||
|
bs58 = { version = "0.5", optional = true }
|
||||||
|
rand_core = { version = "0.6", features = ["getrandom"], optional = true }
|
||||||
|
|
||||||
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
|
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
|
||||||
# cache.rs. Pure-Rust single-process embedded store.
|
# cache.rs. Pure-Rust single-process embedded store.
|
||||||
redb = { version = "2", optional = true }
|
redb = { version = "2", optional = true }
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
A **service is a `Vec`** — grouping plugins into one binary is a deployment
|
A **service is a `Vec`** — grouping plugins into one binary is a deployment
|
||||||
choice, not an architectural one.
|
choice, not an architectural one.
|
||||||
|
|
||||||
### Discovery (1.6.0)
|
### Discovery (1.6.0, scan 1.7.0)
|
||||||
|
|
||||||
`serve_feeders` and `serve_transport` advertise the process over **beacon v2**
|
`serve_feeders` and `serve_transport` advertise the process over **beacon v2**
|
||||||
(UDP multicast `239.255.99.1:9099`, spec: meta-root
|
(UDP multicast `239.255.99.1:9099`, spec: meta-root
|
||||||
@@ -52,11 +52,19 @@ transport as `metamesh.transport/<id>@<contract>` — with `endpoints.http`,
|
|||||||
| `BEACON_BINDS` | unset | the one consumer instance allowed to list this plugin |
|
| `BEACON_BINDS` | unset | the one consumer instance allowed to list this plugin |
|
||||||
| `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS` | `239.255.99.1` / `9099` / `10000` | |
|
| `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS` | `239.255.99.1` / `9099` / `10000` | |
|
||||||
|
|
||||||
|
`beacon::scan(node, pattern, configured, wait)` (1.7.0) is what a consumer's
|
||||||
|
**Scan** button runs: probe everyone, wait, then classify every resource
|
||||||
|
matching `pattern` as `configured` / `addable` / `bound-elsewhere` against the
|
||||||
|
consumer's `(name, url)` list — the `ScanReport` JSON the shared
|
||||||
|
`<meta-beacon-scan>` element renders. Transports advertise as
|
||||||
|
`meta-transport-<id>` (`SERVICE_NAME` overrides), feeders as
|
||||||
|
`meta-feeder-<first upstream>`.
|
||||||
|
|
||||||
The `beacon` module is also usable on its own — the meta-* services depend on
|
The `beacon` module is also usable on its own — the meta-* services depend on
|
||||||
the SDK for nothing else:
|
the SDK for nothing else:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.6.0", default-features = false, features = ["beacon"] }
|
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0", default-features = false, features = ["beacon"] }
|
||||||
```
|
```
|
||||||
|
|
||||||
Features: `feeder` (default — the whole feeder harness; implies `beacon`),
|
Features: `feeder` (default — the whole feeder harness; implies `beacon`),
|
||||||
|
|||||||
@@ -15,8 +15,10 @@
|
|||||||
|
|
||||||
mod node;
|
mod node;
|
||||||
mod proto;
|
mod proto;
|
||||||
|
mod scan;
|
||||||
|
|
||||||
pub use node::{enabled_from_env, hostname, BeaconNode, Config, SeenNode, SeenResource};
|
pub use node::{enabled_from_env, hostname, BeaconNode, Config, SeenNode, SeenResource};
|
||||||
|
pub use scan::{classify, scan, suggested_name, Candidate, CandidateState, ScanReport, ScanSummary};
|
||||||
pub use proto::{
|
pub use proto::{
|
||||||
cap_matches, rev_of, Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT,
|
cap_matches, rev_of, Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT,
|
||||||
LIVENESS_FACTOR, MAX_DATAGRAM, PROTO, VERSION,
|
LIVENESS_FACTOR, MAX_DATAGRAM, PROTO, VERSION,
|
||||||
@@ -38,6 +40,10 @@ pub mod caps {
|
|||||||
pub const FEEDER_PREFIX: &str = "metamesh.feeder/";
|
pub const FEEDER_PREFIX: &str = "metamesh.feeder/";
|
||||||
/// Every feeder upstream.
|
/// Every feeder upstream.
|
||||||
pub const ANY_FEEDER: &str = "metamesh.feeder/*";
|
pub const ANY_FEEDER: &str = "metamesh.feeder/*";
|
||||||
|
/// Prefix of `metamesh.enrich/<plugin_id>` (meta-sort enrichment plugins).
|
||||||
|
pub const ENRICH_PREFIX: &str = "metamesh.enrich/";
|
||||||
|
/// Every meta-sort enrichment plugin.
|
||||||
|
pub const ANY_ENRICH: &str = "metamesh.enrich/*";
|
||||||
|
|
||||||
pub fn service(name: &str) -> String {
|
pub fn service(name: &str) -> String {
|
||||||
format!("{SERVICE_PREFIX}{name}")
|
format!("{SERVICE_PREFIX}{name}")
|
||||||
@@ -50,6 +56,10 @@ pub mod caps {
|
|||||||
pub fn feeder(upstream_id: &str) -> String {
|
pub fn feeder(upstream_id: &str) -> String {
|
||||||
format!("{FEEDER_PREFIX}{upstream_id}")
|
format!("{FEEDER_PREFIX}{upstream_id}")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn enrich(plugin_id: &str) -> String {
|
||||||
|
format!("{ENRICH_PREFIX}{plugin_id}")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The resource every meta-* service advertises: id `service`, cap
|
/// The resource every meta-* service advertises: id `service`, cap
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
//! On-demand discovery scan — what a consumer's "Scan" button runs.
|
||||||
|
//!
|
||||||
|
//! Probe everyone, wait a moment for the replies, then classify every resource
|
||||||
|
//! matching the consumer's capability pattern against what the consumer has
|
||||||
|
//! already configured. The report is the same JSON in every consumer
|
||||||
|
//! (meta-share, meta-gateway, meta-sort), rendered by the shared
|
||||||
|
//! `<meta-beacon-scan>` element.
|
||||||
|
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::node::{BeaconNode, SeenResource};
|
||||||
|
|
||||||
|
/// How a discovered resource relates to this consumer.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum CandidateState {
|
||||||
|
/// Its URL is already in the consumer's list.
|
||||||
|
Configured,
|
||||||
|
/// Can be added.
|
||||||
|
Addable,
|
||||||
|
/// `binds` names another consumer; shown but not addable.
|
||||||
|
BoundElsewhere,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct Candidate {
|
||||||
|
pub instance: String,
|
||||||
|
pub name: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub version: Option<String>,
|
||||||
|
pub resource_id: String,
|
||||||
|
pub caps: Vec<String>,
|
||||||
|
/// `endpoints.http` — what the consumer would register.
|
||||||
|
pub url: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub binds: Option<String>,
|
||||||
|
pub state: CandidateState,
|
||||||
|
/// The list entry that already holds this URL (state `configured`).
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub configured_as: Option<String>,
|
||||||
|
/// A list-safe name to register it under.
|
||||||
|
pub suggested_name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ScanSummary {
|
||||||
|
/// Every node heard on the segment, whatever it advertises.
|
||||||
|
pub nodes: usize,
|
||||||
|
/// Resources matching the consumer's pattern.
|
||||||
|
pub capable: usize,
|
||||||
|
pub configured: usize,
|
||||||
|
pub bound_elsewhere: usize,
|
||||||
|
pub addable: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
pub struct ScanReport {
|
||||||
|
pub cap: String,
|
||||||
|
/// This consumer's instance (what `binds` is compared against).
|
||||||
|
#[serde(rename = "self")]
|
||||||
|
pub self_instance: String,
|
||||||
|
pub duration_ms: u64,
|
||||||
|
pub summary: ScanSummary,
|
||||||
|
pub candidates: Vec<Candidate>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn norm(u: &str) -> String {
|
||||||
|
u.trim().trim_end_matches('/').to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules.
|
||||||
|
pub fn suggested_name(instance: &str) -> String {
|
||||||
|
let s: String = instance
|
||||||
|
.trim()
|
||||||
|
.to_ascii_lowercase()
|
||||||
|
.chars()
|
||||||
|
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '-' })
|
||||||
|
.collect();
|
||||||
|
if s.is_empty() { "plugin".into() } else { s }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pure classification, for tests and for callers that already hold a view.
|
||||||
|
/// `configured` is `(name, url)` of every entry the consumer has.
|
||||||
|
pub fn classify(
|
||||||
|
pattern: &str,
|
||||||
|
self_instance: &str,
|
||||||
|
nodes: usize,
|
||||||
|
resources: Vec<SeenResource>,
|
||||||
|
configured: &[(String, String)],
|
||||||
|
) -> ScanReport {
|
||||||
|
let mut summary = ScanSummary { nodes, ..Default::default() };
|
||||||
|
let mut candidates = Vec::new();
|
||||||
|
for r in resources {
|
||||||
|
// A resource without an http endpoint cannot be registered anywhere.
|
||||||
|
let Some(url) = r.resource.endpoint("http") else { continue };
|
||||||
|
let url = norm(&url);
|
||||||
|
summary.capable += 1;
|
||||||
|
let configured_as = configured.iter().find(|(_, u)| norm(u) == url).map(|(n, _)| n.clone());
|
||||||
|
let state = if configured_as.is_some() {
|
||||||
|
summary.configured += 1;
|
||||||
|
CandidateState::Configured
|
||||||
|
} else if !r.resource.usable_by(self_instance) {
|
||||||
|
summary.bound_elsewhere += 1;
|
||||||
|
CandidateState::BoundElsewhere
|
||||||
|
} else {
|
||||||
|
summary.addable += 1;
|
||||||
|
CandidateState::Addable
|
||||||
|
};
|
||||||
|
candidates.push(Candidate {
|
||||||
|
suggested_name: suggested_name(&r.node.instance),
|
||||||
|
instance: r.node.instance,
|
||||||
|
name: r.node.name,
|
||||||
|
version: r.node.version,
|
||||||
|
resource_id: r.resource.id,
|
||||||
|
caps: r.resource.caps,
|
||||||
|
url,
|
||||||
|
binds: r.resource.binds,
|
||||||
|
state,
|
||||||
|
configured_as,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Addable first (what the operator came for), then configured, then the rest.
|
||||||
|
let rank = |s: CandidateState| match s {
|
||||||
|
CandidateState::Addable => 0,
|
||||||
|
CandidateState::Configured => 1,
|
||||||
|
CandidateState::BoundElsewhere => 2,
|
||||||
|
};
|
||||||
|
candidates.sort_by(|a, b| rank(a.state).cmp(&rank(b.state)).then(a.instance.cmp(&b.instance)));
|
||||||
|
ScanReport {
|
||||||
|
cap: pattern.to_string(),
|
||||||
|
self_instance: self_instance.to_string(),
|
||||||
|
duration_ms: 0,
|
||||||
|
summary,
|
||||||
|
candidates,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe every node, wait `wait` for the replies, then classify what matches
|
||||||
|
/// `pattern`. A node that stays silent through the wait but is still inside
|
||||||
|
/// its liveness window is counted too — the view is "who is here", not "who
|
||||||
|
/// answered this probe".
|
||||||
|
pub async fn scan(
|
||||||
|
node: &BeaconNode,
|
||||||
|
pattern: &str,
|
||||||
|
configured: &[(String, String)],
|
||||||
|
wait: Duration,
|
||||||
|
) -> ScanReport {
|
||||||
|
let started = Instant::now();
|
||||||
|
node.probe(&[]).await;
|
||||||
|
tokio::time::sleep(wait).await;
|
||||||
|
let mut report = classify(
|
||||||
|
pattern,
|
||||||
|
&node.instance(),
|
||||||
|
node.nodes().len(),
|
||||||
|
node.resources(pattern),
|
||||||
|
configured,
|
||||||
|
);
|
||||||
|
report.duration_ms = started.elapsed().as_millis() as u64;
|
||||||
|
report
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::beacon::{NodeInfo, Resource};
|
||||||
|
|
||||||
|
fn seen(instance: &str, cap: &str, url: Option<&str>, binds: Option<&str>) -> SeenResource {
|
||||||
|
let mut r = Resource::new("x", [cap]);
|
||||||
|
if let Some(u) = url {
|
||||||
|
r = r.with_endpoint("http", u);
|
||||||
|
}
|
||||||
|
r.binds = binds.map(Into::into);
|
||||||
|
SeenResource {
|
||||||
|
node: NodeInfo { name: "plugin".into(), instance: instance.into(), ..Default::default() },
|
||||||
|
resource: r,
|
||||||
|
addr: "10.0.0.1".into(),
|
||||||
|
last_seen: 0.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classifies_configured_addable_and_bound_elsewhere() {
|
||||||
|
let resources = vec![
|
||||||
|
seen("metashare-nzb", "metamesh.transport/nzb@1", Some("http://metashare-nzb:3000/"), Some("metashare-app")),
|
||||||
|
seen("qbit", "metamesh.transport/torrent@1", Some("http://qbit:3000"), None),
|
||||||
|
seen("other-ipfs", "metamesh.transport/ipfs@1", Some("http://other-ipfs:3000"), Some("metawatch-share")),
|
||||||
|
seen("no-endpoint", "metamesh.transport/ipfs@1", None, None),
|
||||||
|
];
|
||||||
|
let configured = vec![("nzb".to_string(), "http://metashare-nzb:3000".to_string())];
|
||||||
|
let r = classify("metamesh.transport/*", "metashare-app", 10, resources, &configured);
|
||||||
|
assert_eq!(r.summary.nodes, 10);
|
||||||
|
assert_eq!(r.summary.capable, 3, "a resource without an http endpoint is not a candidate");
|
||||||
|
assert_eq!((r.summary.configured, r.summary.addable, r.summary.bound_elsewhere), (1, 1, 1));
|
||||||
|
assert_eq!(r.candidates[0].instance, "qbit", "addable rows sort first");
|
||||||
|
assert_eq!(r.candidates[1].configured_as.as_deref(), Some("nzb"));
|
||||||
|
assert_eq!(r.candidates[2].state, CandidateState::BoundElsewhere);
|
||||||
|
let json = serde_json::to_value(&r).unwrap();
|
||||||
|
assert_eq!(json["self"], "metashare-app");
|
||||||
|
assert_eq!(json["summary"]["boundElsewhere"], 1);
|
||||||
|
assert_eq!(json["candidates"][2]["state"], "bound-elsewhere");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn suggested_names_are_list_safe() {
|
||||||
|
assert_eq!(suggested_name("MetaShare NZB"), "metashare-nzb");
|
||||||
|
assert_eq!(suggested_name(""), "plugin");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scan_reports_what_the_node_heard() {
|
||||||
|
use crate::beacon::{BeaconNode, Config, Message};
|
||||||
|
let port = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
||||||
|
let mut cfg = Config::new("consumer").instance("me");
|
||||||
|
cfg.port = port;
|
||||||
|
let node = BeaconNode::spawn(cfg).unwrap();
|
||||||
|
let ad = Message::advertise(
|
||||||
|
NodeInfo { name: "meta-feeder-tmdb".into(), instance: "tmdb-feeder".into(), ..Default::default() },
|
||||||
|
vec![Resource::new("feeder", ["metamesh.feeder/tmdb"]).with_endpoint("http", "http://tmdb-feeder:8080")],
|
||||||
|
);
|
||||||
|
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
s.send_to(&serde_json::to_vec(&ad).unwrap(), ("127.0.0.1", port)).await.unwrap();
|
||||||
|
let r = scan(&node, "metamesh.feeder/*", &[], Duration::from_millis(200)).await;
|
||||||
|
assert_eq!(r.summary.nodes, 1);
|
||||||
|
assert_eq!(r.summary.addable, 1);
|
||||||
|
assert_eq!(r.candidates[0].url, "http://tmdb-feeder:8080");
|
||||||
|
assert_eq!(r.candidates[0].suggested_name, "tmdb-feeder");
|
||||||
|
assert!(r.duration_ms >= 200);
|
||||||
|
}
|
||||||
|
}
|
||||||
+602
@@ -0,0 +1,602 @@
|
|||||||
|
//! MMUR2 signed claims: signer `uid` says `key = value` about content `cid`, at
|
||||||
|
//! `ts` (milliseconds, also the version). See meta-root
|
||||||
|
//! `docs/implementation/metacore-claims-model.md` §3.
|
||||||
|
//!
|
||||||
|
//! Wire spec (pinned by `tests/claim-vectors.json`, vendored from meta-core's
|
||||||
|
//! `internal/claim/testdata` — `scripts/check-claim-vectors.sh` guards it):
|
||||||
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! domain = b"MMUR2\n"
|
||||||
|
//! payload = canonical CBOR array [cid, uid, key, value, ts]
|
||||||
|
//! bytes_to_sign = domain || payload
|
||||||
|
//! sig = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! `value` is CBOR text (public), bytes (private envelope) or null (retraction).
|
||||||
|
//! The encoder is hand-rolled: those five shapes are all MMUR2 needs, and a
|
||||||
|
//! hand-rolled head encoder cannot drift from "smallest integer form".
|
||||||
|
//!
|
||||||
|
//! The type and its JSON form are always available with the `feeder` feature
|
||||||
|
//! (the gateway wire carries claims); signing and verification are here too,
|
||||||
|
//! behind `claims`, which `feeder` enables.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use base64::Engine as _;
|
||||||
|
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
|
||||||
|
|
||||||
|
/// Domain separator. Changing it invalidates every claim everywhere.
|
||||||
|
pub const DOMAIN: &[u8] = b"MMUR2\n";
|
||||||
|
|
||||||
|
/// A claim's value. The SQLite storage type in meta-core mirrors it.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||||
|
pub enum ClaimValue {
|
||||||
|
/// Retraction (tombstone).
|
||||||
|
#[default]
|
||||||
|
Null,
|
||||||
|
/// Public plaintext.
|
||||||
|
Text(String),
|
||||||
|
/// Private envelope (XChaCha20-Poly1305), opaque to everyone but the owner.
|
||||||
|
Bytes(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ClaimValue {
|
||||||
|
pub fn is_null(&self) -> bool {
|
||||||
|
matches!(self, ClaimValue::Null)
|
||||||
|
}
|
||||||
|
pub fn as_text(&self) -> Option<&str> {
|
||||||
|
match self {
|
||||||
|
ClaimValue::Text(s) => Some(s),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One signed vote.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct SignedClaim {
|
||||||
|
pub cid: String,
|
||||||
|
pub uid: String,
|
||||||
|
pub key: String,
|
||||||
|
#[serde(with = "value_json")]
|
||||||
|
pub value: ClaimValue,
|
||||||
|
pub ts: u64,
|
||||||
|
/// 64-byte compact r||s, base64 in JSON.
|
||||||
|
#[serde(with = "sig_b64")]
|
||||||
|
pub sig: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SignedClaim {
|
||||||
|
/// The exact bytes the signature covers.
|
||||||
|
pub fn signed_bytes(&self) -> Vec<u8> {
|
||||||
|
signed_bytes(&self.cid, &self.uid, &self.key, &self.value, self.ts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `domain || canonical CBOR [cid, uid, key, value, ts]`.
|
||||||
|
pub fn signed_bytes(cid: &str, uid: &str, key: &str, value: &ClaimValue, ts: u64) -> Vec<u8> {
|
||||||
|
let extra = match value {
|
||||||
|
ClaimValue::Text(s) => s.len(),
|
||||||
|
ClaimValue::Bytes(b) => b.len(),
|
||||||
|
ClaimValue::Null => 0,
|
||||||
|
};
|
||||||
|
let mut b = Vec::with_capacity(DOMAIN.len() + cid.len() + uid.len() + key.len() + extra + 32);
|
||||||
|
b.extend_from_slice(DOMAIN);
|
||||||
|
head(&mut b, 4, 5);
|
||||||
|
text(&mut b, cid);
|
||||||
|
text(&mut b, uid);
|
||||||
|
text(&mut b, key);
|
||||||
|
match value {
|
||||||
|
ClaimValue::Text(s) => text(&mut b, s),
|
||||||
|
ClaimValue::Bytes(bytes) => {
|
||||||
|
head(&mut b, 2, bytes.len() as u64);
|
||||||
|
b.extend_from_slice(bytes);
|
||||||
|
}
|
||||||
|
ClaimValue::Null => b.push(0xf6),
|
||||||
|
}
|
||||||
|
head(&mut b, 0, ts);
|
||||||
|
b
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(b: &mut Vec<u8>, s: &str) {
|
||||||
|
head(b, 3, s.len() as u64);
|
||||||
|
b.extend_from_slice(s.as_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CBOR major type + argument, smallest form.
|
||||||
|
fn head(b: &mut Vec<u8>, major: u8, n: u64) {
|
||||||
|
let m = major << 5;
|
||||||
|
if n < 24 {
|
||||||
|
b.push(m | n as u8);
|
||||||
|
} else if n <= 0xff {
|
||||||
|
b.extend_from_slice(&[m | 24, n as u8]);
|
||||||
|
} else if n <= 0xffff {
|
||||||
|
b.push(m | 25);
|
||||||
|
b.extend_from_slice(&(n as u16).to_be_bytes());
|
||||||
|
} else if n <= 0xffff_ffff {
|
||||||
|
b.push(m | 26);
|
||||||
|
b.extend_from_slice(&(n as u32).to_be_bytes());
|
||||||
|
} else {
|
||||||
|
b.push(m | 27);
|
||||||
|
b.extend_from_slice(&n.to_be_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON (human-readable) uses the spec §3.4 form — a string, `{"b64": …}` or
|
||||||
|
/// `null`. Binary formats (the bincode gateway wire) use a plain tagged enum:
|
||||||
|
/// bincode cannot decode the untagged JSON shape.
|
||||||
|
mod value_json {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct B64 {
|
||||||
|
b64: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
enum Wire {
|
||||||
|
Null,
|
||||||
|
Text(String),
|
||||||
|
Bytes(Vec<u8>),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn serialize<S: Serializer>(v: &ClaimValue, s: S) -> Result<S::Ok, S::Error> {
|
||||||
|
if !s.is_human_readable() {
|
||||||
|
let w = match v {
|
||||||
|
ClaimValue::Null => Wire::Null,
|
||||||
|
ClaimValue::Text(t) => Wire::Text(t.clone()),
|
||||||
|
ClaimValue::Bytes(b) => Wire::Bytes(b.clone()),
|
||||||
|
};
|
||||||
|
return w.serialize(s);
|
||||||
|
}
|
||||||
|
match v {
|
||||||
|
ClaimValue::Null => s.serialize_none(),
|
||||||
|
ClaimValue::Text(t) => s.serialize_str(t),
|
||||||
|
ClaimValue::Bytes(b) => B64 {
|
||||||
|
b64: base64::engine::general_purpose::STANDARD.encode(b),
|
||||||
|
}
|
||||||
|
.serialize(s),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<ClaimValue, D::Error> {
|
||||||
|
if !d.is_human_readable() {
|
||||||
|
return Ok(match Wire::deserialize(d)? {
|
||||||
|
Wire::Null => ClaimValue::Null,
|
||||||
|
Wire::Text(t) => ClaimValue::Text(t),
|
||||||
|
Wire::Bytes(b) => ClaimValue::Bytes(b),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum Raw {
|
||||||
|
Text(String),
|
||||||
|
Bytes(B64),
|
||||||
|
}
|
||||||
|
Ok(match Option::<Raw>::deserialize(d)? {
|
||||||
|
None => ClaimValue::Null,
|
||||||
|
Some(Raw::Text(t)) => ClaimValue::Text(t),
|
||||||
|
Some(Raw::Bytes(B64 { b64 })) => ClaimValue::Bytes(
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(b64)
|
||||||
|
.map_err(de::Error::custom)?,
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// base64 in JSON, raw bytes in binary formats.
|
||||||
|
mod sig_b64 {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
|
||||||
|
if !s.is_human_readable() {
|
||||||
|
// A plain sequence, symmetric with `Vec::<u8>::deserialize` below.
|
||||||
|
return v.serialize(s);
|
||||||
|
}
|
||||||
|
s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
|
||||||
|
if !d.is_human_readable() {
|
||||||
|
return Vec::<u8>::deserialize(d);
|
||||||
|
}
|
||||||
|
let s = String::deserialize(d)?;
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(s)
|
||||||
|
.map_err(de::Error::custom)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Union-flatten a claim list into a field map: every key that has at least
|
||||||
|
/// one non-retracted text vote, with the first such value seen. A
|
||||||
|
/// **superset view for filtering and scoring**, not a merge — the merge (most
|
||||||
|
/// signers → oldest ts → smallest value, own vote first) lives only in
|
||||||
|
/// meta-core (`POST /api/merge`). Ciphertext never appears.
|
||||||
|
pub fn union_fields(claims: &[SignedClaim]) -> BTreeMap<String, String> {
|
||||||
|
let mut out = BTreeMap::new();
|
||||||
|
for c in claims {
|
||||||
|
if let ClaimValue::Text(v) = &c.value {
|
||||||
|
out.entry(c.key.clone()).or_insert_with(|| v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "claims")]
|
||||||
|
pub use crypto::*;
|
||||||
|
|
||||||
|
#[cfg(feature = "claims")]
|
||||||
|
mod crypto {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
use k256::ecdsa::signature::hazmat::{PrehashSigner, PrehashVerifier};
|
||||||
|
use k256::ecdsa::{Signature, SigningKey, VerifyingKey};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
/// Why a claim failed verification.
|
||||||
|
#[derive(Debug, thiserror::Error)]
|
||||||
|
pub enum ClaimError {
|
||||||
|
#[error("claim: missing cid or key")]
|
||||||
|
Shape,
|
||||||
|
#[error("uid: {0}")]
|
||||||
|
Uid(String),
|
||||||
|
#[error("sig: want a 64-byte compact signature")]
|
||||||
|
SigLen,
|
||||||
|
#[error("sig: high-S signatures are refused (MMUR2 is low-S only)")]
|
||||||
|
HighS,
|
||||||
|
#[error("sig: does not verify")]
|
||||||
|
Invalid,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recover the verifying key a uid encodes (`"z" + base58btc(33-byte sec1)`).
|
||||||
|
pub fn pubkey_from_uid(uid: &str) -> Result<VerifyingKey, ClaimError> {
|
||||||
|
let body = uid
|
||||||
|
.strip_prefix('z')
|
||||||
|
.ok_or_else(|| ClaimError::Uid("missing multibase 'z' prefix".into()))?;
|
||||||
|
let raw = bs58::decode(body)
|
||||||
|
.into_vec()
|
||||||
|
.map_err(|e| ClaimError::Uid(e.to_string()))?;
|
||||||
|
if raw.len() != 33 {
|
||||||
|
return Err(ClaimError::Uid(format!(
|
||||||
|
"pubkey is {} bytes, want 33",
|
||||||
|
raw.len()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
VerifyingKey::from_sec1_bytes(&raw).map_err(|e| ClaimError::Uid(e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The uid of a verifying key.
|
||||||
|
pub fn uid_from_pubkey(vk: &VerifyingKey) -> String {
|
||||||
|
let point = vk.to_encoded_point(true);
|
||||||
|
format!("z{}", bs58::encode(point.as_bytes()).into_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SignedClaim {
|
||||||
|
/// Check shape, low-S and the signature against the uid's key.
|
||||||
|
pub fn verify(&self) -> Result<(), ClaimError> {
|
||||||
|
if self.cid.is_empty() || self.key.is_empty() {
|
||||||
|
return Err(ClaimError::Shape);
|
||||||
|
}
|
||||||
|
if self.sig.len() != 64 {
|
||||||
|
return Err(ClaimError::SigLen);
|
||||||
|
}
|
||||||
|
let sig = Signature::from_slice(&self.sig).map_err(|_| ClaimError::Invalid)?;
|
||||||
|
if sig.normalize_s().is_some() {
|
||||||
|
return Err(ClaimError::HighS);
|
||||||
|
}
|
||||||
|
let vk = pubkey_from_uid(&self.uid)?;
|
||||||
|
let digest = Sha256::digest(self.signed_bytes());
|
||||||
|
vk.verify_prehash(&digest, &sig)
|
||||||
|
.map_err(|_| ClaimError::Invalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A writer's identity: one key per writing container (spec D2).
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub struct Signer {
|
||||||
|
key: SigningKey,
|
||||||
|
uid: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for Signer {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("Signer")
|
||||||
|
.field("uid", &self.uid)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Signer {
|
||||||
|
/// From a 32-byte secret, hex (optionally `0x`-prefixed).
|
||||||
|
pub fn from_hex(hex_key: &str) -> anyhow::Result<Self> {
|
||||||
|
let s = hex_key.trim();
|
||||||
|
let s = s
|
||||||
|
.strip_prefix("0x")
|
||||||
|
.or_else(|| s.strip_prefix("0X"))
|
||||||
|
.unwrap_or(s);
|
||||||
|
let raw = decode_hex(s).ok_or_else(|| anyhow::anyhow!("signing key is not hex"))?;
|
||||||
|
if raw.len() != 32 {
|
||||||
|
anyhow::bail!("signing key must be 32 bytes, got {}", raw.len());
|
||||||
|
}
|
||||||
|
let key = SigningKey::from_slice(&raw)
|
||||||
|
.map_err(|e| anyhow::anyhow!("invalid signing key: {e}"))?;
|
||||||
|
let uid = uid_from_pubkey(key.verifying_key());
|
||||||
|
Ok(Signer { key, uid })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A fresh random key.
|
||||||
|
pub fn generate() -> Self {
|
||||||
|
let key = SigningKey::random(&mut rand_core::OsRng);
|
||||||
|
let uid = uid_from_pubkey(key.verifying_key());
|
||||||
|
Signer { key, uid }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn uid(&self) -> &str {
|
||||||
|
&self.uid
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The secret as lowercase hex (for writing a generated key file).
|
||||||
|
pub fn secret_hex(&self) -> String {
|
||||||
|
self.key
|
||||||
|
.to_bytes()
|
||||||
|
.iter()
|
||||||
|
.map(|b| format!("{b:02x}"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign one claim at `ts`. RFC 6979 deterministic, normalised to low-S.
|
||||||
|
pub fn sign(&self, cid: &str, key: &str, value: ClaimValue, ts: u64) -> SignedClaim {
|
||||||
|
let bytes = signed_bytes(cid, &self.uid, key, &value, ts);
|
||||||
|
let digest = Sha256::digest(&bytes);
|
||||||
|
let sig: Signature = self
|
||||||
|
.key
|
||||||
|
.sign_prehash(&digest)
|
||||||
|
.expect("a 32-byte prehash always signs");
|
||||||
|
let sig = sig.normalize_s().unwrap_or(sig);
|
||||||
|
SignedClaim {
|
||||||
|
cid: cid.to_string(),
|
||||||
|
uid: self.uid.clone(),
|
||||||
|
key: key.to_string(),
|
||||||
|
value,
|
||||||
|
ts,
|
||||||
|
sig: sig.to_bytes().to_vec(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign every field of a flat map as text claims on `cid`, all at `ts`.
|
||||||
|
pub fn sign_fields(
|
||||||
|
&self,
|
||||||
|
cid: &str,
|
||||||
|
fields: &BTreeMap<String, String>,
|
||||||
|
ts: u64,
|
||||||
|
) -> Vec<SignedClaim> {
|
||||||
|
fields
|
||||||
|
.iter()
|
||||||
|
.map(|(k, v)| self.sign(cid, k, ClaimValue::Text(v.clone()), ts))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A signed retraction of `key` on `cid`.
|
||||||
|
pub fn retract(&self, cid: &str, key: &str, ts: u64) -> SignedClaim {
|
||||||
|
self.sign(cid, key, ClaimValue::Null, ts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Milliseconds since the epoch — the `ts` of a claim signed now. Strictly
|
||||||
|
/// increasing per process: `ts` is the version, so two writes to one key in
|
||||||
|
/// the same millisecond must not make the second one `stale`.
|
||||||
|
pub fn now_ms() -> u64 {
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
static LAST: AtomicU64 = AtomicU64::new(0);
|
||||||
|
let wall = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_millis() as u64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
let mut last = LAST.load(Ordering::Relaxed);
|
||||||
|
loop {
|
||||||
|
let next = wall.max(last + 1);
|
||||||
|
match LAST.compare_exchange_weak(last, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||||
|
Ok(_) => return next,
|
||||||
|
Err(seen) => last = seen,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Env var holding the signer's secret as hex.
|
||||||
|
pub const SIGNING_KEY_ENV: &str = "METAMESH_SIGNING_KEY";
|
||||||
|
/// Env var naming a key file (bind-mounted); generated on first boot.
|
||||||
|
pub const SIGNING_KEY_FILE_ENV: &str = "METAMESH_SIGNING_KEY_FILE";
|
||||||
|
|
||||||
|
/// The writer's key (spec §9): `METAMESH_SIGNING_KEY` (hex) wins; else
|
||||||
|
/// `METAMESH_SIGNING_KEY_FILE`, generated and written (mode 0600) when the
|
||||||
|
/// file does not exist yet. Neither set → error: a writer must never sign
|
||||||
|
/// with a silent ephemeral identity.
|
||||||
|
pub fn load_or_create_key() -> anyhow::Result<Signer> {
|
||||||
|
if let Some(hex) = env_nonempty(SIGNING_KEY_ENV) {
|
||||||
|
return Signer::from_hex(&hex);
|
||||||
|
}
|
||||||
|
let Some(path) = env_nonempty(SIGNING_KEY_FILE_ENV) else {
|
||||||
|
anyhow::bail!("no signing key: set {SIGNING_KEY_ENV} or {SIGNING_KEY_FILE_ENV}");
|
||||||
|
};
|
||||||
|
load_or_create_key_file(std::path::Path::new(&path))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a hex key file, or generate one there (mode 0600) if absent.
|
||||||
|
pub fn load_or_create_key_file(path: &std::path::Path) -> anyhow::Result<Signer> {
|
||||||
|
match std::fs::read_to_string(path) {
|
||||||
|
Ok(s) => Signer::from_hex(&s),
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||||
|
let signer = Signer::generate();
|
||||||
|
if let Some(dir) = path.parent() {
|
||||||
|
std::fs::create_dir_all(dir)?;
|
||||||
|
}
|
||||||
|
write_secret(path, &signer.secret_hex())?;
|
||||||
|
tracing::info!(target: "meta-feeder::claims", uid = %signer.uid(),
|
||||||
|
path = %path.display(), "generated a new signing key");
|
||||||
|
Ok(signer)
|
||||||
|
}
|
||||||
|
Err(e) => Err(anyhow::anyhow!("read signing key {}: {e}", path.display())),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
|
||||||
|
use std::io::Write;
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
let mut f = std::fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.mode(0o600)
|
||||||
|
.open(path)?;
|
||||||
|
f.write_all(hex.as_bytes())?;
|
||||||
|
f.write_all(b"\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(unix))]
|
||||||
|
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
|
||||||
|
std::fs::write(path, format!("{hex}\n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn env_nonempty(key: &str) -> Option<String> {
|
||||||
|
std::env::var(key)
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_hex(s: &str) -> Option<Vec<u8>> {
|
||||||
|
if s.len() % 2 != 0 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
(0..s.len())
|
||||||
|
.step_by(2)
|
||||||
|
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct Vector {
|
||||||
|
name: String,
|
||||||
|
cid: String,
|
||||||
|
uid: String,
|
||||||
|
key: String,
|
||||||
|
value: Option<String>,
|
||||||
|
#[serde(rename = "valueBytesHex")]
|
||||||
|
value_bytes_hex: Option<String>,
|
||||||
|
ts: u64,
|
||||||
|
#[serde(rename = "signedBytesHex")]
|
||||||
|
signed: String,
|
||||||
|
#[serde(rename = "sigHex")]
|
||||||
|
sig: String,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct HighS {
|
||||||
|
vector: String,
|
||||||
|
#[serde(rename = "sigHex")]
|
||||||
|
sig: String,
|
||||||
|
}
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct File {
|
||||||
|
#[serde(rename = "privateKeyHex")]
|
||||||
|
private_key_hex: String,
|
||||||
|
uid: String,
|
||||||
|
vectors: Vec<Vector>,
|
||||||
|
#[serde(rename = "highS")]
|
||||||
|
high_s: HighS,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hex(b: &[u8]) -> String {
|
||||||
|
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load() -> File {
|
||||||
|
serde_json::from_str(include_str!("../tests/claim-vectors.json")).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn value(v: &Vector) -> ClaimValue {
|
||||||
|
match (&v.value_bytes_hex, &v.value) {
|
||||||
|
(Some(h), _) => ClaimValue::Bytes(decode_hex(h).unwrap()),
|
||||||
|
(None, Some(t)) => ClaimValue::Text(t.clone()),
|
||||||
|
(None, None) => ClaimValue::Null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn golden_vectors() {
|
||||||
|
let f = load();
|
||||||
|
let signer = Signer::from_hex(&f.private_key_hex).unwrap();
|
||||||
|
assert_eq!(signer.uid(), f.uid);
|
||||||
|
for v in &f.vectors {
|
||||||
|
let c = signer.sign(&v.cid, &v.key, value(v), v.ts);
|
||||||
|
assert_eq!(c.uid, v.uid, "{}", v.name);
|
||||||
|
assert_eq!(hex(&c.signed_bytes()), v.signed, "{}: signed bytes", v.name);
|
||||||
|
assert_eq!(hex(&c.sig), v.sig, "{}: RFC 6979 signature", v.name);
|
||||||
|
c.verify().unwrap_or_else(|e| panic!("{}: {e}", v.name));
|
||||||
|
// JSON round trip keeps the claim verifiable.
|
||||||
|
let back: SignedClaim =
|
||||||
|
serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap();
|
||||||
|
assert_eq!(back, c, "{}: json", v.name);
|
||||||
|
}
|
||||||
|
let v = f
|
||||||
|
.vectors
|
||||||
|
.iter()
|
||||||
|
.find(|v| v.name == f.high_s.vector)
|
||||||
|
.unwrap();
|
||||||
|
let high = SignedClaim {
|
||||||
|
cid: v.cid.clone(),
|
||||||
|
uid: v.uid.clone(),
|
||||||
|
key: v.key.clone(),
|
||||||
|
value: value(v),
|
||||||
|
ts: v.ts,
|
||||||
|
sig: decode_hex(&f.high_s.sig).unwrap(),
|
||||||
|
};
|
||||||
|
assert!(matches!(high.verify(), Err(ClaimError::HighS)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tamper_is_refused() {
|
||||||
|
let s = Signer::generate();
|
||||||
|
let mut c = s.sign("bafy", "title", ClaimValue::Text("A".into()), 1);
|
||||||
|
c.value = ClaimValue::Text("B".into());
|
||||||
|
assert!(matches!(c.verify(), Err(ClaimError::Invalid)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn key_file_is_generated_once() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let p = dir.path().join("keys/signing.key");
|
||||||
|
let a = load_or_create_key_file(&p).unwrap();
|
||||||
|
let b = load_or_create_key_file(&p).unwrap();
|
||||||
|
assert_eq!(a.uid(), b.uid());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn json_shapes() {
|
||||||
|
let s = Signer::generate();
|
||||||
|
let j =
|
||||||
|
serde_json::to_value(s.sign("c", "k", ClaimValue::Bytes(vec![1, 2]), 5)).unwrap();
|
||||||
|
assert_eq!(j["value"]["b64"], "AQI=");
|
||||||
|
let j = serde_json::to_value(s.retract("c", "k", 6)).unwrap();
|
||||||
|
assert!(j["value"].is_null());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn union_keeps_first_text_and_skips_retractions_and_cipher() {
|
||||||
|
let s = Signer::generate();
|
||||||
|
let claims = vec![
|
||||||
|
s.sign("c", "title", ClaimValue::Text("A".into()), 1),
|
||||||
|
s.sign("c", "title", ClaimValue::Text("B".into()), 2),
|
||||||
|
s.retract("c", "gone", 3),
|
||||||
|
s.sign("c", "seek", ClaimValue::Bytes(vec![9]), 4),
|
||||||
|
];
|
||||||
|
let f = union_fields(&claims);
|
||||||
|
assert_eq!(f.get("title").map(String::as_str), Some("A"));
|
||||||
|
assert!(!f.contains_key("gone") && !f.contains_key("seek"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+31
-7
@@ -64,6 +64,10 @@ pub struct EnrichmentConfig {
|
|||||||
pub opensubtitles_password: Option<String>,
|
pub opensubtitles_password: Option<String>,
|
||||||
/// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
|
/// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
|
||||||
pub opensubtitles_languages: String,
|
pub opensubtitles_languages: String,
|
||||||
|
/// This feeder's claim signer (`METAMESH_SIGNING_KEY` / `_FILE`). `None`
|
||||||
|
/// when no key is configured: every meta-core write then fails loudly
|
||||||
|
/// instead of publishing unsigned data.
|
||||||
|
pub signer: Option<crate::claims::Signer>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn env_nonempty(key: &str) -> Option<String> {
|
fn env_nonempty(key: &str) -> Option<String> {
|
||||||
@@ -120,6 +124,14 @@ impl EnrichmentConfig {
|
|||||||
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
|
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
|
||||||
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
|
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
|
||||||
.unwrap_or_else(|| "en".to_string()),
|
.unwrap_or_else(|| "en".to_string()),
|
||||||
|
signer: match crate::claims::load_or_create_key() {
|
||||||
|
Ok(s) => Some(s),
|
||||||
|
Err(e) => {
|
||||||
|
warn!(target: "meta-feeder::enrich", error = %e,
|
||||||
|
"no signing key; meta-core writes will fail");
|
||||||
|
None
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -218,7 +230,14 @@ impl Enricher {
|
|||||||
&self.cfg.gateway_peer,
|
&self.cfg.gateway_peer,
|
||||||
);
|
);
|
||||||
let body = meta_core::build_metadata_body(record, &prov);
|
let body = meta_core::build_metadata_body(record, &prov);
|
||||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await
|
meta_core::put_record(&self.http, &self.cfg.meta_core_url, self.signer()?, cid, &body).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn signer(&self) -> anyhow::Result<&crate::claims::Signer> {
|
||||||
|
self.cfg
|
||||||
|
.signer
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| anyhow::anyhow!("no signing key (METAMESH_SIGNING_KEY / _FILE)"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the video enrichment pipeline for one resolved CID: filename-parser
|
/// Run the video enrichment pipeline for one resolved CID: filename-parser
|
||||||
@@ -240,14 +259,14 @@ impl Enricher {
|
|||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
// Poll for the merged fields so tmdb gets the clean title.
|
// Poll for the merged fields so tmdb gets the clean title.
|
||||||
if let Some(merged) = self
|
if let Some(merged) = self
|
||||||
.poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
.poll_until_field(cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
meta = merged;
|
meta = merged;
|
||||||
} else {
|
} else {
|
||||||
debug!(
|
debug!(
|
||||||
target: "meta-feeder::enrich",
|
target: "meta-feeder::enrich",
|
||||||
cid, "filename-parser produced no originalTitle within timeout; \
|
cid, "filename-parser produced no parsedTitle within timeout; \
|
||||||
handing tmdb the base fields"
|
handing tmdb the base fields"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -324,7 +343,7 @@ impl Enricher {
|
|||||||
|
|
||||||
// Poll the primary for filename-parser's clean title, then tmdb it.
|
// Poll the primary for filename-parser's clean title, then tmdb it.
|
||||||
let merged = self
|
let merged = self
|
||||||
.poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
.poll_until_field(&p.cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|| p.base.clone());
|
.unwrap_or_else(|| p.base.clone());
|
||||||
|
|
||||||
@@ -355,9 +374,14 @@ impl Enricher {
|
|||||||
}
|
}
|
||||||
// PUT is merge/upsert — this layers the show-level anchor onto each
|
// PUT is merge/upsert — this layers the show-level anchor onto each
|
||||||
// file's record without disturbing its per-file fields.
|
// file's record without disturbing its per-file fields.
|
||||||
if let Err(e) =
|
let res = match self.signer() {
|
||||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await
|
Ok(signer) => {
|
||||||
{
|
meta_core::put_record(&self.http, &self.cfg.meta_core_url, signer, &v.cid, &anchor)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
Err(e) => Err(e),
|
||||||
|
};
|
||||||
|
if let Err(e) = res {
|
||||||
warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
|
warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -728,6 +728,159 @@ pub fn extract_codec(title: &str) -> Option<String> {
|
|||||||
Some(norm.to_string())
|
Some(norm.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Release facts — `sourceMedium`, `releaseGroup`, `editions/*`, `flags/*`
|
||||||
|
// (METADATA_KEYS §3). Durable properties of a release (unlike its seeders), so
|
||||||
|
// they belong on the record; title-derived and best-effort like `quality` and
|
||||||
|
// `codec`, so absence means "couldn't tell". Each returns a closed vocabulary
|
||||||
|
// and drops what it does not recognise. The same functions normalise a
|
||||||
|
// structured value an upstream already parsed (AIOStreams' `quality`
|
||||||
|
// "BluRay REMUX", its editions "Director's Cut"), so the two paths agree.
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// Separators to spaces, the way every extractor here reads a title. Hyphens
|
||||||
|
/// are kept: `WEB-DL`, `HD-Rip` and the `-GROUP` suffix need them.
|
||||||
|
fn spaced(title: &str) -> String {
|
||||||
|
title.replace(['.', '_'], " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The medium the release was taken from, one of `cam`, `ts`, `tc`, `scr`,
|
||||||
|
/// `bluray-remux`, `bluray`, `webrip`, `web-dl`, `hdrip`, `hdtv`, `dvd`.
|
||||||
|
///
|
||||||
|
/// Pre-release sources are tested first: `HDCAM` beats any `HD` token, and a
|
||||||
|
/// `DVDSCR` is a screener, not a DVD. The bare `CAM`, `TS` and `TC` tokens only
|
||||||
|
/// count in capitals: as words they are a film title (*Cam*, 2018), a file
|
||||||
|
/// extension and an initialism too often.
|
||||||
|
pub fn extract_source_medium(title: &str) -> Option<&'static str> {
|
||||||
|
static CI: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||||
|
[
|
||||||
|
(r"(?i)\b(?:hd-?cam|cam-?rip)\b", "cam"),
|
||||||
|
(r"(?i)\b(?:telesync|hd-?ts|pdvd)\b", "ts"),
|
||||||
|
(r"(?i)\b(?:telecine|hd-?tc)\b", "tc"),
|
||||||
|
(r"(?i)\b(?:dvd-?scr|screener|scr)\b", "scr"),
|
||||||
|
(r"(?i)\b(?:bd-?remux|remux)\b", "bluray-remux"),
|
||||||
|
(r"(?i)\b(?:blu-?ray|bd-?rip|br-?rip|bdmv|bd25|bd50)\b", "bluray"),
|
||||||
|
(r"(?i)\bweb-?rip\b", "webrip"),
|
||||||
|
(r"(?i)\b(?:web-?dl|web)\b", "web-dl"),
|
||||||
|
(r"(?i)\bhd-?rip\b", "hdrip"),
|
||||||
|
(r"(?i)\b(?:hdtv|hdtvrip|uhdtv|pdtv|sdtv|tvrip|dsr)\b", "hdtv"),
|
||||||
|
(r"(?i)\b(?:dvd-?rip|dvd-?r|dvd5|dvd9|dvd)\b", "dvd"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(re, v)| (Regex::new(re).expect("source re compiles"), v))
|
||||||
|
.collect()
|
||||||
|
});
|
||||||
|
static CAPS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||||
|
[(r"\bCAM\b", "cam"), (r"\bTS\b", "ts"), (r"\bTC\b", "tc")]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(re, v)| (Regex::new(re).expect("caps re compiles"), v))
|
||||||
|
.collect()
|
||||||
|
});
|
||||||
|
static DVD_REMUX: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"(?i)\bdvd\S*\b.*\bremux\b|\bremux\b.*\bdvd").expect("dvd remux re"));
|
||||||
|
let t = spaced(title);
|
||||||
|
// A DVD remux is a DVD: `remux` alone names no disc.
|
||||||
|
if DVD_REMUX.is_match(&t) {
|
||||||
|
return Some("dvd");
|
||||||
|
}
|
||||||
|
CAPS.iter()
|
||||||
|
.find(|(re, _)| re.is_match(&t))
|
||||||
|
.or_else(|| CI.iter().find(|(re, _)| re.is_match(&t)))
|
||||||
|
.map(|(_, v)| *v)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The group that made the release: a leading `[Group]` (fansub style) or a
|
||||||
|
/// trailing `-GROUP` (scene style), the same two rules as meta-watch's
|
||||||
|
/// `parseVersion`. Trailing tracker tags (`[rarbg]`, `[TGx]`), a video
|
||||||
|
/// extension and a usenet `-xpost` are removed first; a candidate that is
|
||||||
|
/// really a format token (`WEB-DL` → `DL`, `x264`, `1080p`) is refused.
|
||||||
|
pub fn extract_release_group(title: &str) -> Option<String> {
|
||||||
|
static LEADING: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"^\s*\[([^\[\]]{2,32})\]").expect("leading group re"));
|
||||||
|
static TRAILING: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"-\s*([A-Za-z0-9][A-Za-z0-9&@]{1,24})\s*$").expect("trailing group re"));
|
||||||
|
static TAIL_NOISE: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
|
Regex::new(r"(?i)(?:\s*(?:\[[^\]]*\]|\([^)]*\))|\.(?:mkv|mp4|avi|m4v|ts|m2ts|webm|wmv|mov)|-xpost)+\s*$")
|
||||||
|
.expect("tail noise re")
|
||||||
|
});
|
||||||
|
static FORMAT: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
|
Regex::new(r"^(?:[xh]26[45]|hevc|avc|av1|\d{3,4}[pi]|s\d+(?:e\d+)?)$").expect("format re")
|
||||||
|
});
|
||||||
|
let not_a_format = |g: &str| {
|
||||||
|
let l = g.to_ascii_lowercase();
|
||||||
|
!(matches!(
|
||||||
|
l.as_str(),
|
||||||
|
"dl" | "rip" | "web" | "hd" | "hdr" | "dts" | "ac3" | "aac" | "remux" | "sample" | "eng" | "multi"
|
||||||
|
) || l.chars().all(|c| c.is_ascii_digit())
|
||||||
|
|| FORMAT.is_match(&l))
|
||||||
|
};
|
||||||
|
if let Some(g) = LEADING.captures(title).map(|c| c[1].trim().to_string()) {
|
||||||
|
// `[1080p]` or `[HEVC]` at the front is a tag, not a group.
|
||||||
|
if not_a_format(&g) && extract_quality(&g).is_none() {
|
||||||
|
return Some(g);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut t = title.trim().to_string();
|
||||||
|
loop {
|
||||||
|
let cut = TAIL_NOISE.replace(&t, "").trim_end().to_string();
|
||||||
|
if cut == t {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
t = cut;
|
||||||
|
}
|
||||||
|
TRAILING
|
||||||
|
.captures(&t)
|
||||||
|
.map(|c| c[1].to_string())
|
||||||
|
.filter(|g| not_a_format(g))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cut of the work this release carries — `editions/<x>` members. A
|
||||||
|
/// different cut has a different runtime, so these matter beyond a label
|
||||||
|
/// (a subtitle synced to the theatrical cut drifts on the extended one).
|
||||||
|
pub fn extract_editions(title: &str) -> BTreeSet<&'static str> {
|
||||||
|
static EDITIONS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||||
|
[
|
||||||
|
(r"(?i)\bextended\b", "extended"),
|
||||||
|
(r"(?i)\bdirector'?s?[\s-]*cut\b", "directors-cut"),
|
||||||
|
(r"(?i)\btheatrical\b", "theatrical"),
|
||||||
|
(r"(?i)\bunrated\b", "unrated"),
|
||||||
|
(r"(?i)\buncut\b", "uncut"),
|
||||||
|
(r"(?i)\buncensored\b", "uncensored"),
|
||||||
|
(r"(?i)\bimax\b", "imax"),
|
||||||
|
(r"(?i)\bremaster(?:ed)?\b", "remastered"),
|
||||||
|
(r"(?i)\bcriterion\b", "criterion"),
|
||||||
|
(r"(?i)\banniversary\b", "anniversary"),
|
||||||
|
(r"(?i)\bcollector'?s\b", "collectors"),
|
||||||
|
(r"(?i)\bultimate[\s-]+(?:edition|cut)\b", "ultimate"),
|
||||||
|
(r"(?i)\bspecial[\s-]+edition\b", "special"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(re, v)| (Regex::new(re).expect("edition re compiles"), v))
|
||||||
|
.collect()
|
||||||
|
});
|
||||||
|
let t = spaced(title);
|
||||||
|
EDITIONS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Properties of the encode itself — `flags/<x>` members: `repack` and
|
||||||
|
/// `proper` (a fixed re-release that supersedes the first), `upscaled` (the
|
||||||
|
/// resolution is not the source's: a 2160 upscale is not 4K), `regraded`.
|
||||||
|
pub fn extract_flags(title: &str) -> BTreeSet<&'static str> {
|
||||||
|
static FLAGS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||||
|
[
|
||||||
|
(r"(?i)\b(?:repack\d?|rerip)\b", "repack"),
|
||||||
|
(r"(?i)\bproper\b", "proper"),
|
||||||
|
(r"(?i)\b(?:ai[\s-]*)?upscaled?\b", "upscaled"),
|
||||||
|
(r"(?i)\bregraded?\b", "regraded"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(re, v)| (Regex::new(re).expect("flag re compiles"), v))
|
||||||
|
.collect()
|
||||||
|
});
|
||||||
|
let t = spaced(title);
|
||||||
|
FLAGS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
|
/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
|
||||||
/// `cleanSpace`, in that order.
|
/// `cleanSpace`, in that order.
|
||||||
fn preclean(filename: &str) -> String {
|
fn preclean(filename: &str) -> String {
|
||||||
@@ -935,6 +1088,61 @@ pub fn clean_title(raw: &str) -> String {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn source_medium_reads_real_release_names() {
|
||||||
|
let m = extract_source_medium;
|
||||||
|
assert_eq!(m("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR"), Some("bluray-remux"));
|
||||||
|
assert_eq!(m("The.Matrix.1999.1080p.BrRip.x264.YIFY.mp4"), Some("bluray"));
|
||||||
|
assert_eq!(m("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv"), Some("web-dl"));
|
||||||
|
assert_eq!(m("Show.S01E01.1080p.WEB.H264-GROUP"), Some("web-dl"));
|
||||||
|
assert_eq!(m("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), Some("webrip"));
|
||||||
|
assert_eq!(m("Movie 2023 HC HD-Rip 720p"), Some("hdrip"));
|
||||||
|
assert_eq!(m("Some Show [2x10] HDTV"), Some("hdtv"));
|
||||||
|
assert_eq!(m("Old.Film.1985.DVDRip.XviD"), Some("dvd"));
|
||||||
|
assert_eq!(m("Old.Film.1985.DVD.REMUX"), Some("dvd"));
|
||||||
|
assert_eq!(m("New.Film.2024.HDCAM.x264"), Some("cam"));
|
||||||
|
assert_eq!(m("New Film 2024 CAM"), Some("cam"));
|
||||||
|
assert_eq!(m("New.Film.2024.DVDSCR.x264"), Some("scr"));
|
||||||
|
assert_eq!(m("New.Film.2024.TELESYNC"), Some("ts"));
|
||||||
|
// AIOStreams' structured vocabulary goes through the same door.
|
||||||
|
assert_eq!(m("BluRay REMUX"), Some("bluray-remux"));
|
||||||
|
assert_eq!(m("WEB-DL"), Some("web-dl"));
|
||||||
|
assert_eq!(m("TC"), Some("tc"));
|
||||||
|
// The film "Cam" and a lower-case "ts" are not sources.
|
||||||
|
assert_eq!(m("Cam (2018) 1080p"), None);
|
||||||
|
assert_eq!(m("its a ts file"), None);
|
||||||
|
assert_eq!(m("Movie 2019 1080p x264"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn release_group_is_the_leading_bracket_or_trailing_dash() {
|
||||||
|
let g = |t: &str| extract_release_group(t);
|
||||||
|
assert_eq!(g("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)").as_deref(), Some("yolerejiju"));
|
||||||
|
assert_eq!(g("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv").as_deref(), Some("G66"));
|
||||||
|
assert_eq!(g("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR").as_deref(), Some("FraMeSToR"));
|
||||||
|
assert_eq!(g("Oppenheimer.2023.1080p.BluRay.DD5.1.x264-GalaxyRG.mkv").as_deref(), Some("GalaxyRG"));
|
||||||
|
assert_eq!(g("Movie.2020.1080p.WEB-DL.x264-NTb[rarbg]").as_deref(), Some("NTb"));
|
||||||
|
assert_eq!(g("Breaking.Bad.S02E04.Down.1080p.BluRay.x264-OFT.mkv-xpost").as_deref(), Some("OFT"));
|
||||||
|
assert_eq!(g("Movie 2020 1080p WEB-DL [TGx]").as_deref(), None, "WEB-DL is not a group called DL");
|
||||||
|
assert_eq!(g("[1080p] Movie - 2020").as_deref(), None);
|
||||||
|
assert_eq!(g("Movie.2020.1080p.x264").as_deref(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn editions_and_flags() {
|
||||||
|
let e = |t: &str| extract_editions(t).into_iter().collect::<Vec<_>>();
|
||||||
|
assert_eq!(e("Aliens.1986.Directors.Cut.1080p.BluRay"), ["directors-cut"]);
|
||||||
|
assert_eq!(e("LOTR.Fellowship.2001.EXTENDED.REMASTERED.2160p"), ["extended", "remastered"]);
|
||||||
|
assert_eq!(e("Director's Cut"), ["directors-cut"]);
|
||||||
|
assert_eq!(e("Ultimate Edition"), ["ultimate"]);
|
||||||
|
assert_eq!(e("Movie.2019.1080p.WEB-DL"), Vec::<&str>::new());
|
||||||
|
let f = |t: &str| extract_flags(t).into_iter().collect::<Vec<_>>();
|
||||||
|
assert_eq!(f("Game of Thrones S01E03 Lord Snow REPACK 1080p"), ["repack"]);
|
||||||
|
assert_eq!(f("Show.S01E01.PROPER.720p.HDTV"), ["proper"]);
|
||||||
|
assert_eq!(f("Movie 1999 2160p AI Upscaled Regrade"), ["regraded", "upscaled"]);
|
||||||
|
assert_eq!(f("The Proper Way 2020"), ["proper"], "a title word is indistinguishable — accepted noise");
|
||||||
|
}
|
||||||
|
|
||||||
fn se(title: &str) -> (Option<String>, Option<String>) {
|
fn se(title: &str) -> (Option<String>, Option<String>) {
|
||||||
let r = extract_season_episode(title);
|
let r = extract_season_episode(title);
|
||||||
(r.season, r.episode)
|
(r.season, r.episode)
|
||||||
|
|||||||
+9
-1
@@ -34,6 +34,8 @@ pub mod beacon;
|
|||||||
|
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod budget;
|
pub mod budget;
|
||||||
|
#[cfg(any(feature = "feeder", feature = "claims"))]
|
||||||
|
pub mod claims;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod cache;
|
pub mod cache;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
@@ -44,7 +46,7 @@ pub mod config;
|
|||||||
pub mod domain;
|
pub mod domain;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod enrich;
|
pub mod enrich;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "filename")]
|
||||||
pub mod filename_meta;
|
pub mod filename_meta;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod hash;
|
pub mod hash;
|
||||||
@@ -52,6 +54,8 @@ pub mod hash;
|
|||||||
pub mod lang;
|
pub mod lang;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod licence;
|
pub mod licence;
|
||||||
|
/// Search-wire-transient `live/*` fields — always available, no dependencies.
|
||||||
|
pub mod live;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub mod meta_core;
|
pub mod meta_core;
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
@@ -90,5 +94,9 @@ pub use serve::{
|
|||||||
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
|
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
|
||||||
RedeemsResponse,
|
RedeemsResponse,
|
||||||
};
|
};
|
||||||
|
#[cfg(any(feature = "feeder", feature = "claims"))]
|
||||||
|
pub use claims::{ClaimValue, SignedClaim};
|
||||||
|
#[cfg(feature = "claims")]
|
||||||
|
pub use claims::{load_or_create_key, ClaimError, Signer};
|
||||||
#[cfg(feature = "feeder")]
|
#[cfg(feature = "feeder")]
|
||||||
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
||||||
|
|||||||
+79
@@ -0,0 +1,79 @@
|
|||||||
|
//! Search-wire-transient fields: facts a feeder sees at query time that are
|
||||||
|
//! true only *now* — a torrent's seeders, its leechers — and so must never
|
||||||
|
//! reach the replicated `/file/{cid}` hash (METADATA_KEYS rule #4).
|
||||||
|
//!
|
||||||
|
//! They travel under one reserved prefix, [`LIVE_PREFIX`], so every layer that
|
||||||
|
//! persists a record strips the whole family without knowing each key:
|
||||||
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! live/seeders = "42" int-string
|
||||||
|
//! live/leechers = "3" int-string
|
||||||
|
//! live/at = "1791726110" unix seconds the feeder observed them
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! A record without them means "unknown", never "zero": a replayed or
|
||||||
|
//! persisted record has none, and that is the honest answer.
|
||||||
|
//!
|
||||||
|
//! The feeder SDK strips them in [`crate::meta_core::build_metadata_body`]
|
||||||
|
//! (every feeder self-publish); meta-gateway and meta-search strip them on
|
||||||
|
//! their own write paths (see METADATA_KEYS "Search-wire-transient fields").
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
/// The reserved prefix of the transient family.
|
||||||
|
pub const LIVE_PREFIX: &str = "live/";
|
||||||
|
|
||||||
|
/// Swarm seeders, `int-string`.
|
||||||
|
pub const LIVE_SEEDERS: &str = "live/seeders";
|
||||||
|
/// Swarm leechers, `int-string`.
|
||||||
|
pub const LIVE_LEECHERS: &str = "live/leechers";
|
||||||
|
/// When the feeder observed the `live/*` values, unix seconds.
|
||||||
|
pub const LIVE_AT: &str = "live/at";
|
||||||
|
|
||||||
|
/// The bare swarm counters some feeders wrote before the `live/*` family
|
||||||
|
/// existed (tribler). Stripped with it: rule #4 bars them in any spelling.
|
||||||
|
pub const LEGACY_SWARM_KEYS: [&str; 3] = ["seeders", "leechers", "peers"];
|
||||||
|
|
||||||
|
/// Whether a field is search-wire-transient and must not be persisted.
|
||||||
|
pub fn is_transient(key: &str) -> bool {
|
||||||
|
key.starts_with(LIVE_PREFIX) || LEGACY_SWARM_KEYS.contains(&key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove every transient field. Call on the copy about to be persisted, never
|
||||||
|
/// on the record still bound for the search wire.
|
||||||
|
pub fn strip_transient(fields: &mut BTreeMap<String, String>) {
|
||||||
|
fields.retain(|k, _| !is_transient(k));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `live/at` value for a feeder observing counters right now.
|
||||||
|
pub fn now_secs() -> String {
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn strips_the_family_and_the_legacy_counters_only() {
|
||||||
|
let mut f: BTreeMap<String, String> = [
|
||||||
|
("live/seeders", "42"),
|
||||||
|
("live/at", "1"),
|
||||||
|
("seeders", "7"),
|
||||||
|
("peers", "9"),
|
||||||
|
("fileName", "x.mkv"),
|
||||||
|
("livestream", "true"),
|
||||||
|
("source/live", "true"),
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||||
|
.collect();
|
||||||
|
strip_transient(&mut f);
|
||||||
|
let keys: Vec<&str> = f.keys().map(String::as_str).collect();
|
||||||
|
assert_eq!(keys, ["fileName", "livestream", "source/live"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
+119
-21
@@ -12,10 +12,12 @@
|
|||||||
//! written by the feeder is indistinguishable from one the dispatcher wrote —
|
//! written by the feeder is indistinguishable from one the dispatcher wrote —
|
||||||
//! the meta-share ingester, meta-watch, and ranking all behave the same.
|
//! the meta-share ingester, meta-watch, and ranking all behave the same.
|
||||||
//!
|
//!
|
||||||
//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core
|
//! Since meta-core v2.0.0 every write is a batch of **signed claims** to
|
||||||
//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the
|
//! `POST /api/claims` (spec `docs/implementation/metacore-claims-model.md`):
|
||||||
//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}`
|
//! each field becomes one MMUR2 vote signed by this feeder's own key. Votes from
|
||||||
//! merges accumulate on the same `/file/{cid}` hash.
|
//! different writers never overwrite each other — meta-core merges them — so the
|
||||||
|
//! feeder's base write and the enrichment plugins' later claims accumulate on
|
||||||
|
//! the same cid.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
@@ -23,6 +25,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
|
use crate::claims::{now_ms, ClaimValue, SignedClaim, Signer};
|
||||||
use crate::plugin::{HashKind, HashOutcome};
|
use crate::plugin::{HashKind, HashOutcome};
|
||||||
use crate::types::{DiscoveryRecord, Hash};
|
use crate::types::{DiscoveryRecord, Hash};
|
||||||
|
|
||||||
@@ -80,6 +83,11 @@ pub fn build_metadata_body(
|
|||||||
) -> BTreeMap<String, String> {
|
) -> BTreeMap<String, String> {
|
||||||
let mut body: BTreeMap<String, String> = BTreeMap::new();
|
let mut body: BTreeMap<String, String> = BTreeMap::new();
|
||||||
for (k, v) in &record.fields {
|
for (k, v) in &record.fields {
|
||||||
|
// `live/*` (and the legacy bare swarm counters) are true only at query
|
||||||
|
// time; a self-publishing feeder must not freeze them into the hash.
|
||||||
|
if crate::live::is_transient(k) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if k.starts_with("cid_") && !v.is_empty() {
|
if k.starts_with("cid_") && !v.is_empty() {
|
||||||
body.insert(format!("cids/{v}"), "true".to_string());
|
body.insert(format!("cids/{v}"), "true".to_string());
|
||||||
} else {
|
} else {
|
||||||
@@ -90,22 +98,76 @@ pub fn build_metadata_body(
|
|||||||
body
|
body
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into
|
/// One per-claim verdict from `POST /api/claims`.
|
||||||
/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`).
|
#[derive(Debug, Deserialize)]
|
||||||
pub async fn put_record(
|
pub struct ClaimStatus {
|
||||||
|
pub status: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub reason: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/claims` — submit already-signed claims (e.g. relayed verbatim).
|
||||||
|
/// Fails on a non-2xx answer or when any claim is refused as `bad_signature` /
|
||||||
|
/// `bad_request`; `stale` and `unchanged` are normal outcomes.
|
||||||
|
pub async fn post_claims(
|
||||||
http: &reqwest::Client,
|
http: &reqwest::Client,
|
||||||
base_url: &str,
|
base_url: &str,
|
||||||
cid: &str,
|
claims: &[SignedClaim],
|
||||||
body: &BTreeMap<String, String>,
|
) -> anyhow::Result<Vec<ClaimStatus>> {
|
||||||
) -> anyhow::Result<()> {
|
if claims.is_empty() {
|
||||||
let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid);
|
return Ok(Vec::new());
|
||||||
let resp = http.put(&url).json(body).send().await?;
|
}
|
||||||
|
let url = format!("{}/api/claims", base_url.trim_end_matches('/'));
|
||||||
|
let resp = http.post(&url).json(claims).send().await?;
|
||||||
let status = resp.status();
|
let status = resp.status();
|
||||||
if !status.is_success() {
|
if !status.is_success() {
|
||||||
let text = resp.text().await.unwrap_or_default();
|
let text = resp.text().await.unwrap_or_default();
|
||||||
anyhow::bail!("meta-core PUT {url} returned {status}: {text}");
|
anyhow::bail!("meta-core POST {url} returned {status}: {text}");
|
||||||
}
|
}
|
||||||
Ok(())
|
let verdicts: Vec<ClaimStatus> = resp.json().await?;
|
||||||
|
if let Some((i, bad)) = verdicts
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.find(|(_, v)| v.status == "bad_signature" || v.status == "bad_request")
|
||||||
|
{
|
||||||
|
anyhow::bail!(
|
||||||
|
"meta-core refused claim {} ({}): {} {}",
|
||||||
|
i,
|
||||||
|
claims.get(i).map(|c| c.key.as_str()).unwrap_or("?"),
|
||||||
|
bad.status,
|
||||||
|
bad.reason.as_deref().unwrap_or("")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(verdicts)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign every field of `body` as a text claim on `cid` (all at "now") and
|
||||||
|
/// submit them. The claims-model replacement for the v1 `PUT /api/metadata`.
|
||||||
|
pub async fn put_record(
|
||||||
|
http: &reqwest::Client,
|
||||||
|
base_url: &str,
|
||||||
|
signer: &Signer,
|
||||||
|
cid: &str,
|
||||||
|
body: &BTreeMap<String, String>,
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let claims = signer.sign_fields(cid, body, now_ms());
|
||||||
|
post_claims(http, base_url, &claims).await.map(|_| ())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sign retractions of `keys` on `cid` (this signer's own votes only).
|
||||||
|
pub async fn retract_keys(
|
||||||
|
http: &reqwest::Client,
|
||||||
|
base_url: &str,
|
||||||
|
signer: &Signer,
|
||||||
|
cid: &str,
|
||||||
|
keys: &[String],
|
||||||
|
) -> anyhow::Result<()> {
|
||||||
|
let ts = now_ms();
|
||||||
|
let claims: Vec<SignedClaim> = keys
|
||||||
|
.iter()
|
||||||
|
.map(|k| signer.sign(cid, k, ClaimValue::Null, ts))
|
||||||
|
.collect();
|
||||||
|
post_claims(http, base_url, &claims).await.map(|_| ())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`).
|
/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`).
|
||||||
@@ -157,27 +219,49 @@ pub struct FeederStore {
|
|||||||
http: reqwest::Client,
|
http: reqwest::Client,
|
||||||
meta_core_url: String,
|
meta_core_url: String,
|
||||||
gateway_peer: String,
|
gateway_peer: String,
|
||||||
|
signer: Signer,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl FeederStore {
|
impl FeederStore {
|
||||||
pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self {
|
pub fn new(
|
||||||
|
http: reqwest::Client,
|
||||||
|
meta_core_url: String,
|
||||||
|
gateway_peer: String,
|
||||||
|
signer: Signer,
|
||||||
|
) -> Self {
|
||||||
FeederStore {
|
FeederStore {
|
||||||
http,
|
http,
|
||||||
meta_core_url: meta_core_url.trim_end_matches('/').to_string(),
|
meta_core_url: meta_core_url.trim_end_matches('/').to_string(),
|
||||||
gateway_peer,
|
gateway_peer,
|
||||||
|
signer,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build from the feeder env. `None` when `META_CORE_URL` is unset (the
|
/// Build from the feeder env. `None` when `META_CORE_URL` is unset, or when
|
||||||
/// feeder then falls back to the legacy "return outcomes, gateway stores"
|
/// no signing key is configured (`METAMESH_SIGNING_KEY` /
|
||||||
/// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`.
|
/// `METAMESH_SIGNING_KEY_FILE`) — the feeder then falls back to the legacy
|
||||||
|
/// "return outcomes, gateway stores" path. `gateway_peer` ←
|
||||||
|
/// `META_GATEWAY_PEER_ID` / `HOSTNAME`.
|
||||||
pub fn from_env() -> Option<Self> {
|
pub fn from_env() -> Option<Self> {
|
||||||
let meta_core_url = env_nonempty("META_CORE_URL")?;
|
let meta_core_url = env_nonempty("META_CORE_URL")?;
|
||||||
|
let signer = match crate::claims::load_or_create_key() {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
warn!(target: "meta-feeder::store", error = %e,
|
||||||
|
"META_CORE_URL set but no signing key; not self-publishing");
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID")
|
let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID")
|
||||||
.or_else(|| env_nonempty("HOSTNAME"))
|
.or_else(|| env_nonempty("HOSTNAME"))
|
||||||
.unwrap_or_else(|| "gateway-feeder".to_string());
|
.unwrap_or_else(|| "gateway-feeder".to_string());
|
||||||
let http = crate::common::build_http_client(30, "meta-feeder-store", None);
|
let http = crate::common::build_http_client(30, "meta-feeder-store", None);
|
||||||
Some(Self::new(http, meta_core_url, gateway_peer))
|
Some(Self::new(http, meta_core_url, gateway_peer, signer))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The uid this store signs as.
|
||||||
|
pub fn signer(&self) -> &Signer {
|
||||||
|
&self.signer
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn meta_core_url(&self) -> &str {
|
pub fn meta_core_url(&self) -> &str {
|
||||||
@@ -257,7 +341,8 @@ impl FeederStore {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance.
|
/// Metadata-only self-publish: the record's fields (+ provenance) as signed
|
||||||
|
/// claims to `POST /api/claims`.
|
||||||
async fn store_metadata_only(
|
async fn store_metadata_only(
|
||||||
&self,
|
&self,
|
||||||
upstream_id: &str,
|
upstream_id: &str,
|
||||||
@@ -267,7 +352,7 @@ impl FeederStore {
|
|||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer);
|
let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer);
|
||||||
let body = build_metadata_body(record, &prov);
|
let body = build_metadata_body(record, &prov);
|
||||||
put_record(&self.http, &self.meta_core_url, cid, &body).await
|
put_record(&self.http, &self.meta_core_url, &self.signer, cid, &body).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -300,6 +385,7 @@ mod tests {
|
|||||||
("title".into(), "Big Buck Bunny".into()),
|
("title".into(), "Big Buck Bunny".into()),
|
||||||
("cid_btih_v1_file".into(), "bafkPERFILE".into()),
|
("cid_btih_v1_file".into(), "bafkPERFILE".into()),
|
||||||
]),
|
]),
|
||||||
|
claims: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,4 +418,16 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert!(body.contains_key("provenance"));
|
assert!(body.contains_key("provenance"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A self-publish never freezes swarm counters, in either spelling.
|
||||||
|
#[test]
|
||||||
|
fn body_drops_transient_fields() {
|
||||||
|
let mut r = rec();
|
||||||
|
for (k, v) in [("live/seeders", "42"), ("live/at", "1"), ("seeders", "7"), ("leechers", "2")] {
|
||||||
|
r.fields.insert(k.into(), v.into());
|
||||||
|
}
|
||||||
|
let body = build_metadata_body(&r, &Provenance::now("tribler", "abc", "peer"));
|
||||||
|
assert!(body.keys().all(|k| !crate::live::is_transient(k)), "{body:?}");
|
||||||
|
assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,6 +100,10 @@ pub enum GatewaySearchEvent {
|
|||||||
record_id: String,
|
record_id: String,
|
||||||
set: BTreeMap<String, String>,
|
set: BTreeMap<String, String>,
|
||||||
remove: Vec<String>,
|
remove: Vec<String>,
|
||||||
|
/// Signed form of the patch: `set` as text claims, `remove` as
|
||||||
|
/// retractions. Empty from a feeder (the gateway core signs).
|
||||||
|
#[serde(default)]
|
||||||
|
claims: Vec<crate::claims::SignedClaim>,
|
||||||
},
|
},
|
||||||
/// Retract a previously-emitted `Base` record entirely.
|
/// Retract a previously-emitted `Base` record entirely.
|
||||||
Drop { record_id: String },
|
Drop { record_id: String },
|
||||||
|
|||||||
@@ -799,6 +799,7 @@ mod tests {
|
|||||||
upstream_id: "stub".into(),
|
upstream_id: "stub".into(),
|
||||||
record_id: "1".into(),
|
record_id: "1".into(),
|
||||||
fields: BTreeMap::new(),
|
fields: BTreeMap::new(),
|
||||||
|
claims: Vec::new(),
|
||||||
};
|
};
|
||||||
let events = vec![
|
let events = vec![
|
||||||
GatewaySearchEvent::Base(rec),
|
GatewaySearchEvent::Base(rec),
|
||||||
@@ -806,6 +807,7 @@ mod tests {
|
|||||||
record_id: "1".into(),
|
record_id: "1".into(),
|
||||||
set: BTreeMap::from([("title".to_string(), "X".to_string())]),
|
set: BTreeMap::from([("title".to_string(), "X".to_string())]),
|
||||||
remove: vec![],
|
remove: vec![],
|
||||||
|
claims: vec![],
|
||||||
},
|
},
|
||||||
GatewaySearchEvent::Done,
|
GatewaySearchEvent::Done,
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -156,6 +156,25 @@ impl RemoteTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `PUT /manifests/:cid`: hand a manifest-consuming plugin (contract 2) the
|
||||||
|
/// manifest a pointer resolved to, after it answered `428` +
|
||||||
|
/// [`super::dto::HDR_NEEDS_MANIFEST`].
|
||||||
|
pub async fn push_manifest(&self, cid: &str, manifest: bytes::Bytes) -> Result<(), ApiError> {
|
||||||
|
let r = self
|
||||||
|
.http
|
||||||
|
.put(self.url(&format!("/manifests/{cid}")))
|
||||||
|
.body(manifest)
|
||||||
|
.timeout(self.control_timeout)
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| self.unreachable(e))?;
|
||||||
|
if r.status().is_success() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(error_from(r).await)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---- typed helpers for extra routes ------------------------------------
|
// ---- typed helpers for extra routes ------------------------------------
|
||||||
|
|
||||||
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
|
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
|
||||||
|
|||||||
+31
-2
@@ -9,8 +9,37 @@ use serde::{Deserialize, Serialize};
|
|||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
/// Bumped on any breaking change to the common routes below. The hull refuses
|
/// Bumped on any breaking change to the common routes below. The hull refuses
|
||||||
/// a plugin whose `/manifest` reports a different major contract.
|
/// a plugin whose `/manifest` reports a contract outside
|
||||||
pub const CONTRACT_VERSION: u32 = 1;
|
/// [`MIN_CONTRACT`]`..=`[`CONTRACT_VERSION`].
|
||||||
|
///
|
||||||
|
/// - **2**: a plugin that consumes a manifest (nzb) no longer pulls it from the
|
||||||
|
/// hull. It answers `/raw` (and its verify route) with `428` +
|
||||||
|
/// [`HDR_NEEDS_MANIFEST`] until the hull has pushed one with
|
||||||
|
/// `PUT /manifests/:cid`.
|
||||||
|
pub const CONTRACT_VERSION: u32 = 2;
|
||||||
|
/// The oldest contract the hull still drives (contract-1 nzb plugins pull their
|
||||||
|
/// manifest from `/internal/nzb/manifest/:cid`, kept for one release).
|
||||||
|
pub const MIN_CONTRACT: u32 = 1;
|
||||||
|
|
||||||
|
/// `428` marker: this plugin needs the cid's manifest pushed before it can
|
||||||
|
/// answer. The hull resolves the pointer, `PUT`s `/manifests/:cid`, and retries.
|
||||||
|
pub const HDR_NEEDS_MANIFEST: &str = "x-metamesh-needs-manifest";
|
||||||
|
|
||||||
|
/// Why a pointer could not be resolved, on a non-2xx `/ipfs-tier/resolve`
|
||||||
|
/// answer: [`RESOLVE_NO_GATEWAY`], [`RESOLVE_UNCLAIMED`], [`RESOLVE_QUOTA`],
|
||||||
|
/// [`RESOLVE_NOT_FOUND`] or [`RESOLVE_UPSTREAM`].
|
||||||
|
pub const HDR_RESOLVE_ERROR: &str = "x-metamesh-resolve-error";
|
||||||
|
/// No gateway is known at all yet (discovery is eventual): retry.
|
||||||
|
pub const RESOLVE_NO_GATEWAY: &str = "no-gateway";
|
||||||
|
/// Gateways are known, but none claims this pointer's key.
|
||||||
|
pub const RESOLVE_UNCLAIMED: &str = "unclaimed";
|
||||||
|
/// The provider's quota is spent: retry after `Retry-After`.
|
||||||
|
pub const RESOLVE_QUOTA: &str = "quota";
|
||||||
|
/// Every claiming gateway answered not found (or nothing is stored and the
|
||||||
|
/// pointer cannot be redeemed).
|
||||||
|
pub const RESOLVE_NOT_FOUND: &str = "not-found";
|
||||||
|
/// A gateway failed, or answered bytes that did not verify.
|
||||||
|
pub const RESOLVE_UPSTREAM: &str = "upstream";
|
||||||
|
|
||||||
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
|
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
|
||||||
pub const HDR_LANE: &str = "x-metamesh-lane";
|
pub const HDR_LANE: &str = "x-metamesh-lane";
|
||||||
|
|||||||
@@ -39,10 +39,11 @@ use super::dto::Event;
|
|||||||
/// Default hull callback base, matching the dev and store compose service name.
|
/// Default hull callback base, matching the dev and store compose service name.
|
||||||
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
|
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
|
||||||
|
|
||||||
/// Where the hull got a manifest: the record's `manifest` pointer, or a
|
/// Where a pointer's bytes came from: `store` (already held here, named by the
|
||||||
/// gateway redeem.
|
/// record's pointer field), `redeem` (a gateway), or `record` (contract-1
|
||||||
|
/// callback naming for `store`).
|
||||||
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
|
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
|
||||||
/// The content cid of a `record`-sourced manifest.
|
/// The content cid the pointer resolved to.
|
||||||
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
|
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
|
||||||
|
|
||||||
/// `.nzb` bytes as the hull resolved them.
|
/// `.nzb` bytes as the hull resolved them.
|
||||||
|
|||||||
@@ -17,6 +17,7 @@
|
|||||||
//! |---|---|
|
//! |---|---|
|
||||||
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
|
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
|
||||||
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
|
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
|
||||||
|
//! | `GET resolve/:cid?hint=` → bytes | a [pointer](super::pointer)'s bytes: the stored copy `hint` names, else a gateway redeem (see below) |
|
||||||
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
|
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
|
||||||
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
|
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
|
||||||
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
|
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
|
||||||
@@ -40,6 +41,15 @@
|
|||||||
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
|
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
|
||||||
//! a display name.
|
//! a display name.
|
||||||
|
|
||||||
|
//! **Resolving a pointer** (`GET resolve/:cid`) is the one route that reaches
|
||||||
|
//! outside the swarm: a pointer is answered only by a gateway holding the key,
|
||||||
|
//! over plain HTTP at the gateway's base URL. Gateways come from config (the
|
||||||
|
//! local one, pinned) and from the swarm directory; the fastest one whose redeem
|
||||||
|
//! claim covers the pointer is asked first. `2xx` carries the bytes plus
|
||||||
|
//! [`HDR_MANIFEST_SOURCE`](super::hull::HDR_MANIFEST_SOURCE) (`store` | `redeem`)
|
||||||
|
//! and [`HDR_MANIFEST_CID`](super::hull::HDR_MANIFEST_CID); a failure carries
|
||||||
|
//! [`HDR_RESOLVE_ERROR`](super::dto::HDR_RESOLVE_ERROR).
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
|
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
|
||||||
|
|||||||
@@ -22,6 +22,7 @@
|
|||||||
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
|
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
|
||||||
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
|
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
|
||||||
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
|
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
|
||||||
|
//! - [`pointer`]: which cids are pointers (resolved before their bytes exist).
|
||||||
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
|
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
|
||||||
//! - `testkit` (feature `testkit`): conformance checks for any implementation.
|
//! - `testkit` (feature `testkit`): conformance checks for any implementation.
|
||||||
//!
|
//!
|
||||||
@@ -42,6 +43,7 @@ pub mod ipfs;
|
|||||||
pub mod magic;
|
pub mod magic;
|
||||||
pub mod nzb;
|
pub mod nzb;
|
||||||
pub mod plugin;
|
pub mod plugin;
|
||||||
|
pub mod pointer;
|
||||||
pub mod range;
|
pub mod range;
|
||||||
pub mod serve;
|
pub mod serve;
|
||||||
pub mod torrent;
|
pub mod torrent;
|
||||||
@@ -52,7 +54,7 @@ pub use client::RemoteTransport;
|
|||||||
pub use config::ConfigPlane;
|
pub use config::ConfigPlane;
|
||||||
pub use dto::{
|
pub use dto::{
|
||||||
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
|
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
|
||||||
ReconcileRequest, CONTRACT_VERSION,
|
ReconcileRequest, CONTRACT_VERSION, MIN_CONTRACT,
|
||||||
};
|
};
|
||||||
pub use error::ApiError;
|
pub use error::ApiError;
|
||||||
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
|
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
//! **Pointer cids** — cids that name something to fetch rather than hash it.
|
||||||
|
//!
|
||||||
|
//! A pointer's multihash is `identity`: the cid *contains* the address (an
|
||||||
|
//! indexer host + release id, a provider source + file id), not a digest of the
|
||||||
|
//! bytes. Bitswap cannot answer one, because there is no hash to verify against.
|
||||||
|
//! It is resolved instead: by a manifest this peer already stored (the record's
|
||||||
|
//! pointer field names its content cid), or by a **redeem** at a gateway whose
|
||||||
|
//! feeder holds the credential for `key`.
|
||||||
|
//!
|
||||||
|
//! This module is the single place that knows which codecs are pointers, so the
|
||||||
|
//! hull ("resolve it first") and the ipfs plugin ("here is how") agree.
|
||||||
|
//!
|
||||||
|
//! | codec | what it points at | redeem key | record field |
|
||||||
|
//! |---|---|---|---|
|
||||||
|
//! | `0x1005` nzb-release | a `.nzb` at an indexer | indexer host | `manifest` |
|
||||||
|
//! | `0x1003` nzb-posting | a self-scanned `.nzb` | — (record only) | `manifest` |
|
||||||
|
//! | `0x100A` provider-file | a file a provider holds | provider source | `file` |
|
||||||
|
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
|
||||||
|
use super::cid::{parse_record_cid, read_uvarint};
|
||||||
|
use super::nzb::{
|
||||||
|
decode_nzb_release_cid, normalize_indexer_host, NZB_POSTING_CODEC, NZB_RELEASE_CODEC,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// MetaMesh-private multicodec for a provider-held file (`docs/cid-formats.md` §8).
|
||||||
|
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
|
||||||
|
|
||||||
|
/// Redeem-claim codec names, as gateways advertise them in
|
||||||
|
/// `GET /api/gateway/plugins` (`redeems[].codec`).
|
||||||
|
pub const REDEEM_NZB_RELEASE: &str = "nzb-release";
|
||||||
|
pub const REDEEM_PROVIDER_FILE: &str = "provider-file";
|
||||||
|
|
||||||
|
/// Record fields a redeem merges onto the pointer's record: the content cid of
|
||||||
|
/// what it resolved to.
|
||||||
|
pub const FIELD_MANIFEST: &str = "manifest";
|
||||||
|
pub const FIELD_FILE: &str = "file";
|
||||||
|
|
||||||
|
/// Identity multihash code: the digest bytes are the payload verbatim.
|
||||||
|
const IDENTITY_MULTIHASH: u64 = 0x00;
|
||||||
|
|
||||||
|
/// A decoded pointer.
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct Pointer {
|
||||||
|
/// The redeem-claim codec, or `None` when no gateway can redeem it (a
|
||||||
|
/// self-scanned posting resolves from the record alone).
|
||||||
|
pub redeem: Option<RedeemKey>,
|
||||||
|
/// The record field naming the resolved content cid.
|
||||||
|
pub field: &'static str,
|
||||||
|
/// What the resolved bytes are: a manifest a byte plugin consumes, or the
|
||||||
|
/// file itself.
|
||||||
|
pub kind: PointerKind,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct RedeemKey {
|
||||||
|
/// [`REDEEM_NZB_RELEASE`] or [`REDEEM_PROVIDER_FILE`].
|
||||||
|
pub codec: &'static str,
|
||||||
|
/// The indexer host / provider source a gateway's claim must cover.
|
||||||
|
pub key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum PointerKind {
|
||||||
|
/// A `.nzb`: handed to the nzb plugin, which fetches the bytes.
|
||||||
|
NzbManifest,
|
||||||
|
/// The file itself (a subtitle): served as is.
|
||||||
|
File,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode `cid` as a pointer.
|
||||||
|
///
|
||||||
|
/// - `None` — not a pointer: an ordinary content cid, or unparseable.
|
||||||
|
/// - `Some(Err(_))` — the codec says pointer but the payload is malformed; the
|
||||||
|
/// caller answers `400`.
|
||||||
|
pub fn decode(cid: &str) -> Option<Result<Pointer>> {
|
||||||
|
match codec_of(cid)? {
|
||||||
|
NZB_RELEASE_CODEC => Some(decode_nzb_release(cid)),
|
||||||
|
NZB_POSTING_CODEC => Some(Ok(Pointer {
|
||||||
|
redeem: None,
|
||||||
|
field: FIELD_MANIFEST,
|
||||||
|
kind: PointerKind::NzbManifest,
|
||||||
|
})),
|
||||||
|
PROVIDER_FILE_CODEC => Some(decode_provider_file(cid).map(|(source, _id)| Pointer {
|
||||||
|
redeem: Some(RedeemKey { codec: REDEEM_PROVIDER_FILE, key: source }),
|
||||||
|
field: FIELD_FILE,
|
||||||
|
kind: PointerKind::File,
|
||||||
|
})),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is `cid` a pointer (well-formed or not)?
|
||||||
|
pub fn is_pointer(cid: &str) -> bool {
|
||||||
|
decode(cid).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode_nzb_release(cid: &str) -> Result<Pointer> {
|
||||||
|
let mscid = parse_record_cid(cid)?;
|
||||||
|
let locator = decode_nzb_release_cid(&mscid)?;
|
||||||
|
Ok(Pointer {
|
||||||
|
redeem: Some(RedeemKey {
|
||||||
|
codec: REDEEM_NZB_RELEASE,
|
||||||
|
key: normalize_indexer_host(locator.authority()),
|
||||||
|
}),
|
||||||
|
field: FIELD_MANIFEST,
|
||||||
|
kind: PointerKind::NzbManifest,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode a `provider-file` cid into `(source, id)`. Errors on a different
|
||||||
|
/// codec or a malformed payload. Works on the raw multibase string because a
|
||||||
|
/// long provider id can exceed the 64-byte multihash a typed cid parse allows.
|
||||||
|
pub fn decode_provider_file(cid: &str) -> Result<(String, String)> {
|
||||||
|
let bytes = raw_cid_bytes(cid).context("provider-file: not a base32 CIDv1")?;
|
||||||
|
let mut at = 0usize;
|
||||||
|
let next = |at: &mut usize, what: &str| -> Result<u64> {
|
||||||
|
let (v, n) = read_uvarint(&bytes[*at..])
|
||||||
|
.with_context(|| format!("provider-file: {what} varint"))?;
|
||||||
|
*at += n;
|
||||||
|
Ok(v)
|
||||||
|
};
|
||||||
|
anyhow::ensure!(next(&mut at, "version")? == 1, "provider-file: not a CIDv1");
|
||||||
|
anyhow::ensure!(next(&mut at, "codec")? == PROVIDER_FILE_CODEC, "not a provider-file cid");
|
||||||
|
let mh = next(&mut at, "multihash-code")?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
mh == IDENTITY_MULTIHASH,
|
||||||
|
"provider-file locator must use the identity multihash (0x00), got 0x{mh:x}"
|
||||||
|
);
|
||||||
|
let len = next(&mut at, "digest-length")?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
(bytes.len() - at) as u64 == len,
|
||||||
|
"provider-file digest length {len} != {} trailing byte(s)",
|
||||||
|
bytes.len() - at
|
||||||
|
);
|
||||||
|
let source_len = next(&mut at, "source-length")?;
|
||||||
|
let rest = &bytes[at..];
|
||||||
|
let source_len = usize::try_from(source_len)
|
||||||
|
.ok()
|
||||||
|
.filter(|n| *n <= rest.len())
|
||||||
|
.context("provider-file: source length overruns the digest")?;
|
||||||
|
let (source, id) = rest.split_at(source_len);
|
||||||
|
let source = std::str::from_utf8(source).context("provider-file source is not utf-8")?;
|
||||||
|
let id = std::str::from_utf8(id).context("provider-file id is not utf-8")?;
|
||||||
|
anyhow::ensure!(
|
||||||
|
!source.is_empty() && !id.is_empty(),
|
||||||
|
"provider-file locator needs both a source and an id"
|
||||||
|
);
|
||||||
|
Ok((source.to_string(), id.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The codec of a base32 CIDv1 string, without the 64-byte multihash limit.
|
||||||
|
fn codec_of(cid: &str) -> Option<u64> {
|
||||||
|
let bytes = raw_cid_bytes(cid)?;
|
||||||
|
let (version, n) = read_uvarint(&bytes).ok()?;
|
||||||
|
if version != 1 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
read_uvarint(&bytes[n..]).ok().map(|(codec, _)| codec)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decode a `b`-prefixed lowercase base32 (RFC 4648, no padding) CID string.
|
||||||
|
fn raw_cid_bytes(cid: &str) -> Option<Vec<u8>> {
|
||||||
|
let body = cid.strip_prefix('b')?;
|
||||||
|
let mut out = Vec::with_capacity(body.len() * 5 / 8);
|
||||||
|
let (mut acc, mut bits) = (0u32, 0u32);
|
||||||
|
for c in body.bytes() {
|
||||||
|
let v = match c {
|
||||||
|
b'a'..=b'z' => c - b'a',
|
||||||
|
b'2'..=b'7' => c - b'2' + 26,
|
||||||
|
_ => return None,
|
||||||
|
} as u32;
|
||||||
|
acc = (acc << 5) | v;
|
||||||
|
bits += 5;
|
||||||
|
if bits >= 8 {
|
||||||
|
bits -= 8;
|
||||||
|
out.push((acc >> bits) as u8);
|
||||||
|
acc &= (1 << bits) - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The Ragna Crimson release from watch.nsl.sh (2026-10-04).
|
||||||
|
const NZBGEEK: &str = "bagcsaabfcrqxa2jonz5gez3fmvvs42lomzxs6ylqngxuch22nhfhl4v4vmyhiegah2ka";
|
||||||
|
|
||||||
|
fn b32(bytes: &[u8]) -> String {
|
||||||
|
const A: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
|
||||||
|
let (mut acc, mut bits, mut s) = (0u32, 0u32, String::from("b"));
|
||||||
|
for &b in bytes {
|
||||||
|
acc = (acc << 8) | b as u32;
|
||||||
|
bits += 8;
|
||||||
|
while bits >= 5 {
|
||||||
|
bits -= 5;
|
||||||
|
s.push(A[((acc >> bits) & 31) as usize] as char);
|
||||||
|
}
|
||||||
|
acc &= (1 << bits) - 1;
|
||||||
|
}
|
||||||
|
if bits > 0 {
|
||||||
|
s.push(A[((acc << (5 - bits)) & 31) as usize] as char);
|
||||||
|
}
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
fn provider_file(source: &str, id: &str) -> String {
|
||||||
|
let mut digest = vec![source.len() as u8];
|
||||||
|
digest.extend_from_slice(source.as_bytes());
|
||||||
|
digest.extend_from_slice(id.as_bytes());
|
||||||
|
// version 1, codec 0x100A (varint 0x8a 0x20), identity, length
|
||||||
|
let mut wire = vec![0x01, 0x8a, 0x20, 0x00, digest.len() as u8];
|
||||||
|
wire.extend_from_slice(&digest);
|
||||||
|
b32(&wire)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_nzb_release_is_a_pointer_keyed_by_its_indexer_host() {
|
||||||
|
let p = decode(NZBGEEK).unwrap().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
p.redeem,
|
||||||
|
Some(RedeemKey { codec: REDEEM_NZB_RELEASE, key: "api.nzbgeek.info".into() })
|
||||||
|
);
|
||||||
|
assert_eq!(p.field, FIELD_MANIFEST);
|
||||||
|
assert_eq!(p.kind, PointerKind::NzbManifest);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_provider_file_is_a_pointer_keyed_by_its_source() {
|
||||||
|
let cid = provider_file("opensubtitles", "file:7061834");
|
||||||
|
assert_eq!(
|
||||||
|
decode_provider_file(&cid).unwrap(),
|
||||||
|
("opensubtitles".into(), "file:7061834".into())
|
||||||
|
);
|
||||||
|
let p = decode(&cid).unwrap().unwrap();
|
||||||
|
assert_eq!(p.redeem.unwrap().key, "opensubtitles");
|
||||||
|
assert_eq!(p.field, FIELD_FILE);
|
||||||
|
assert_eq!(p.kind, PointerKind::File);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_content_cid_is_not_a_pointer() {
|
||||||
|
assert!(!is_pointer("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy"));
|
||||||
|
assert!(!is_pointer("not a cid"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -66,7 +66,13 @@ pub async fn serve_transport<P: TransportPlugin>(
|
|||||||
// Beacon v2: advertise once bound; say `bye` on clean shutdown so the hull
|
// Beacon v2: advertise once bound; say `bye` on clean shutdown so the hull
|
||||||
// drops us at once instead of after the liveness window.
|
// drops us at once instead of after the liveness window.
|
||||||
let port = listen.port();
|
let port = listen.port();
|
||||||
let beacon = crate::beacon::advertise_plugin(&manifest.implementation, &manifest.version, move || {
|
// `node.name` is a short machine name; the human description stays in the
|
||||||
|
// manifest (`implementation`), which a consumer fetches over HTTP.
|
||||||
|
let beacon_name = std::env::var("SERVICE_NAME")
|
||||||
|
.ok()
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.unwrap_or_else(|| format!("meta-transport-{}", manifest.id));
|
||||||
|
let beacon = crate::beacon::advertise_plugin(&beacon_name, &manifest.version, move || {
|
||||||
beacon_resource(advertised.as_ref(), port)
|
beacon_resource(advertised.as_ref(), port)
|
||||||
});
|
});
|
||||||
axum::serve(listener, app)
|
axum::serve(listener, app)
|
||||||
|
|||||||
@@ -49,6 +49,13 @@ pub struct DiscoveryRecord {
|
|||||||
pub upstream_id: String,
|
pub upstream_id: String,
|
||||||
pub record_id: String,
|
pub record_id: String,
|
||||||
pub fields: BTreeMap<String, String>,
|
pub fields: BTreeMap<String, String>,
|
||||||
|
/// The record's MMUR2 claims (spec §10). Empty from a feeder that returns
|
||||||
|
/// plain `fields` (the gateway core signs those as itself); filled by a
|
||||||
|
/// self-publishing feeder or by the gateway before the record goes on the
|
||||||
|
/// wire. `#[serde(default)]` keeps older feeders' JSON `/query` answers
|
||||||
|
/// decoding.
|
||||||
|
#[serde(default)] // never skip_serializing_if: bincode is positional
|
||||||
|
pub claims: Vec<crate::claims::SignedClaim>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// In-process error returned by `FeederPlugin` methods.
|
/// In-process error returned by `FeederPlugin` methods.
|
||||||
@@ -102,6 +109,7 @@ mod tests {
|
|||||||
("title".to_string(), "AlphaFold".to_string()),
|
("title".to_string(), "AlphaFold".to_string()),
|
||||||
("year".to_string(), "2021".to_string()),
|
("year".to_string(), "2021".to_string()),
|
||||||
]),
|
]),
|
||||||
|
claims: Vec::new(),
|
||||||
};
|
};
|
||||||
let j = serde_json::to_string(&r).expect("serialize");
|
let j = serde_json::to_string(&r).expect("serialize");
|
||||||
let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize");
|
let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize");
|
||||||
|
|||||||
@@ -0,0 +1,93 @@
|
|||||||
|
{
|
||||||
|
"_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
|
||||||
|
"privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"vectors": [
|
||||||
|
{
|
||||||
|
"name": "text",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "title",
|
||||||
|
"value": "The Matrix",
|
||||||
|
"ts": 1793001234567,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087",
|
||||||
|
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "empty-text",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "title",
|
||||||
|
"value": "",
|
||||||
|
"ts": 1,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001",
|
||||||
|
"sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "unicode",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "titles/jpn/攻殻機動隊",
|
||||||
|
"value": "true",
|
||||||
|
"ts": 1793001234567,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087",
|
||||||
|
"sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "keyset-member",
|
||||||
|
"cid": "bagacbabaec7v3",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "cids/bafkreih5kapbjzq",
|
||||||
|
"value": "true",
|
||||||
|
"ts": 23,
|
||||||
|
"signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517",
|
||||||
|
"sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "bytes",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "seek",
|
||||||
|
"value": null,
|
||||||
|
"valueBytesHex": "00ff10ab",
|
||||||
|
"ts": 24,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818",
|
||||||
|
"sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "retraction",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "genres/Drama",
|
||||||
|
"value": null,
|
||||||
|
"ts": 255,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff",
|
||||||
|
"sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "long-value",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "plot/eng",
|
||||||
|
"value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||||
|
"ts": 65536,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000",
|
||||||
|
"sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ts-64bit",
|
||||||
|
"cid": "bafkreih5kapbjzq",
|
||||||
|
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||||
|
"key": "year",
|
||||||
|
"value": "1999",
|
||||||
|
"ts": 1099511627776,
|
||||||
|
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000",
|
||||||
|
"sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"highS": {
|
||||||
|
"note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it",
|
||||||
|
"vector": "text",
|
||||||
|
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user