6 Commits
23 changed files with 3091 additions and 56 deletions
+48 -15
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "meta-feeder-sdk" name = "meta-feeder-sdk"
version = "1.5.0" version = "1.10.0"
edition = "2021" edition = "2021"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
@@ -32,44 +32,77 @@ publish = false
# of here so feeders that don't need them don't pay for them. # of here so feeders that don't need them don't pay for them.
[features] [features]
default = ["feeder"]
# Everything a feeder sidecar (and the transport contract) builds on: the
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
# so existing consumers build exactly what they always did.
feeder = [
"beacon", "filename", "claims",
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
"dep:redb",
]
# Release-name parsing (`src/filename_meta.rs`: clean_title, season/episode,
# quality, codec). Needs only `regex`, so a client (meta-watch) can derive a
# display name from a raw release string without the feeder machinery.
filename = ["dep:regex"]
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
# so a meta-* service takes the SDK with `default-features = false,
# features = ["beacon"]` and pulls in none of the feeder machinery.
beacon = ["dep:libc"]
# The meta-share transport-plugin contract (`src/transport/`): the hull's # The meta-share transport-plugin contract (`src/transport/`): the hull's
# client, the plugins' axum harness, the focus replica, shared error/range # client, the plugins' axum harness, the focus replica, shared error/range
# types. Off by default so feeders build exactly what they always did. # types. Off by default so feeders build exactly what they always did.
transport = ["dep:cid", "dep:libc", "dep:tokio-util"] transport = ["feeder", "dep:cid", "dep:tokio-util"]
# Conformance checks any transport implementation must pass. # Conformance checks any transport implementation must pass.
testkit = ["transport"] testkit = ["transport"]
# MMUR2 signed claims (`src/claims.rs`): the `SignedClaim` type, its canonical
# bytes, signing (k256, compact low-S), verification and the writer key
# (`METAMESH_SIGNING_KEY` / `_FILE`). `feeder` enables it because the gateway
# wire (`DiscoveryRecord`, `EnrichPatch`) carries claims; a meta-* service that
# only needs claims takes `default-features = false, features = ["claims"]`.
claims = [
"dep:k256", "dep:bs58", "dep:sha2", "dep:base64", "dep:rand_core",
"dep:anyhow", "dep:thiserror",
]
[dependencies] [dependencies]
# Async runtime + HTTP server (the serve_feeders harness) # Async runtime + HTTP server (the serve_feeders harness)
tokio = { version = "1.40", features = ["full"] } tokio = { version = "1.40", features = ["full"] }
axum = { version = "0.7", features = ["json"] } axum = { version = "0.7", features = ["json"], optional = true }
hyper = { version = "1", features = ["full"] } hyper = { version = "1", features = ["full"], optional = true }
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false, optional = true }
# Serialization # Serialization
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
base64 = "0.22" base64 = { version = "0.22", optional = true }
# Utilities # Utilities
futures = "0.3" futures = { version = "0.3", optional = true }
async-trait = "0.1" async-trait = { version = "0.1", optional = true }
anyhow = "1" anyhow = { version = "1", optional = true }
thiserror = "1" thiserror = { version = "1", optional = true }
tracing = "0.1" tracing = "0.1"
sha2 = "0.10" sha2 = { version = "0.10", optional = true }
bytes = "1" bytes = { version = "1", optional = true }
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). # Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
regex = "1" regex = { version = "1", optional = true }
# MMUR2 claims (the `claims` feature).
k256 = { version = "0.13", features = ["ecdsa", "std"], optional = true }
bs58 = { version = "0.5", optional = true }
rand_core = { version = "0.6", features = ["getrandom"], optional = true }
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see # Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
# cache.rs. Pure-Rust single-process embedded store. # cache.rs. Pure-Rust single-process embedded store.
redb = "2" redb = { version = "2", optional = true }
# CID parsing for the transport contract only (`transport::cid`). Optional so # CID parsing for the transport contract only (`transport::cid`). Optional so
# feeders don't pull it. # feeders don't pull it.
cid = { version = "0.11", optional = true } cid = { version = "0.11", optional = true }
# statvfs(2) for `transport::fsutil::available_bytes`. # Raw sockets for `beacon` (per-interface multicast) and statvfs(2) for
# `transport::fsutil::available_bytes`.
libc = { version = "0.2", optional = true } libc = { version = "0.2", optional = true }
# `ReaderStream` for `transport::file`. # `ReaderStream` for `transport::file`.
tokio-util = { version = "0.7", features = ["io"], optional = true } tokio-util = { version = "0.7", features = ["io"], optional = true }
+37
View File
@@ -35,6 +35,42 @@ async fn main() -> anyhow::Result<()> {
A **service is a `Vec`** — grouping plugins into one binary is a deployment A **service is a `Vec`** — grouping plugins into one binary is a deployment
choice, not an architectural one. choice, not an architectural one.
### Discovery (1.6.0, scan 1.7.0)
`serve_feeders` and `serve_transport` advertise the process over **beacon v2**
(UDP multicast `239.255.99.1:9099`, spec: meta-root
`docs/project-architecture/beacon-v2.md`) as soon as the port is bound — a
feeder as `metamesh.feeder/<upstream_id>` (one cap per hosted plugin), a
transport as `metamesh.transport/<id>@<contract>` — with `endpoints.http`,
`endpoints.manifest = /manifest` and `rev` = a hash of the served manifest
(re-advertised when it changes). Nothing to call. Env:
| Env | Default | |
|---|---|---|
| `ENABLE_UDP_DISCOVERY` | `true` | `false` turns advertising off |
| `BEACON_ADVERTISE_URL` | `http://$HOSTNAME:<port>` | `endpoints.http` |
| `BEACON_BINDS` | unset | the one consumer instance allowed to list this plugin |
| `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS` | `239.255.99.1` / `9099` / `10000` | |
`beacon::scan(node, pattern, configured, wait)` (1.7.0) is what a consumer's
**Scan** button runs: probe everyone, wait, then classify every resource
matching `pattern` as `configured` / `addable` / `bound-elsewhere` against the
consumer's `(name, url)` list — the `ScanReport` JSON the shared
`<meta-beacon-scan>` element renders. Transports advertise as
`meta-transport-<id>` (`SERVICE_NAME` overrides), feeders as
`meta-feeder-<first upstream>`.
The `beacon` module is also usable on its own — the meta-* services depend on
the SDK for nothing else:
```toml
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0", default-features = false, features = ["beacon"] }
```
Features: `feeder` (default — the whole feeder harness; implies `beacon`),
`beacon` (tokio/serde/tracing/libc only), `transport` (implies `feeder`),
`testkit`.
## Writing a plugin ## Writing a plugin
Four required methods; everything else has a default: Four required methods; everything else has a default:
@@ -141,6 +177,7 @@ and `provider-file` (`0x100A`, `(source, id)` — `docs/cid-formats.md` §8).
| `hash` | the content-addressing families above | | `hash` | the content-addressing families above |
| `filename_meta` / `lang` | release-tag parsing helpers | | `filename_meta` / `lang` | release-tag parsing helpers |
| `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract | | `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract |
| `beacon` | beacon v2 local discovery: `BeaconNode`, `Resource`, `cap_matches`, the MetaMesh `caps` vocabulary |
## Development ## Development
+165
View File
@@ -0,0 +1,165 @@
//! Beacon v2 — local resource advertise / discover over UDP multicast
//! (`239.255.99.1:9099`).
//!
//! Every meta-* service and every SDK-hosted plugin runs one [`BeaconNode`]: it
//! advertises this process's [`Resource`]s, each tagged with capability strings
//! (`metamesh.service/meta-sort`, `metamesh.transport/nzb@1`, …), and keeps a
//! live view of everyone else's, which consumers filter by capability
//! ([`cap_matches`]).
//!
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`
//! (the Go / TypeScript / Python ports are written against it too). This module
//! needs only the `beacon` feature, so a service can depend on the SDK with
//! `default-features = false, features = ["beacon"]` and pull in none of the
//! feeder machinery.
mod node;
mod proto;
mod scan;
pub use node::{enabled_from_env, hostname, BeaconNode, Config, SeenNode, SeenResource};
pub use scan::{classify, scan, suggested_name, Candidate, CandidateState, ScanReport, ScanSummary};
pub use proto::{
cap_matches, rev_of, Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT,
LIVENESS_FACTOR, MAX_DATAGRAM, PROTO, VERSION,
};
/// Capability vocabulary of the MetaMesh profile (spec Part 2).
pub mod caps {
/// meta-core; its resource carries `data.urls`.
pub const CORE: &str = "metamesh.core";
/// Prefix of `metamesh.service/<name>`.
pub const SERVICE_PREFIX: &str = "metamesh.service/";
/// Every service.
pub const ANY_SERVICE: &str = "metamesh.service/*";
/// Prefix of `metamesh.transport/<id>@<contract>`.
pub const TRANSPORT_PREFIX: &str = "metamesh.transport/";
/// Every transport plugin.
pub const ANY_TRANSPORT: &str = "metamesh.transport/*";
/// Prefix of `metamesh.feeder/<upstream_id>`.
pub const FEEDER_PREFIX: &str = "metamesh.feeder/";
/// Every feeder upstream.
pub const ANY_FEEDER: &str = "metamesh.feeder/*";
/// Prefix of `metamesh.enrich/<plugin_id>` (meta-sort enrichment plugins).
pub const ENRICH_PREFIX: &str = "metamesh.enrich/";
/// Every meta-sort enrichment plugin.
pub const ANY_ENRICH: &str = "metamesh.enrich/*";
pub fn service(name: &str) -> String {
format!("{SERVICE_PREFIX}{name}")
}
pub fn transport(id: &str, contract: u32) -> String {
format!("{TRANSPORT_PREFIX}{id}@{contract}")
}
pub fn feeder(upstream_id: &str) -> String {
format!("{FEEDER_PREFIX}{upstream_id}")
}
pub fn enrich(plugin_id: &str) -> String {
format!("{ENRICH_PREFIX}{plugin_id}")
}
}
/// The resource every meta-* service advertises: id `service`, cap
/// `metamesh.service/<name>`, `endpoints.ui` = its browser-facing URL.
pub fn service_resource(name: &str, ui_url: Option<&str>) -> Resource {
let mut r = Resource::new("service", [caps::service(name)]);
if let Some(u) = ui_url {
r = r.with_endpoint("ui", u);
}
r
}
/// `BEACON_ADVERTISE_URL`, else `http://$HOSTNAME:<port>` — what a plugin
/// advertises as `endpoints.http`.
pub fn advertise_url(listen_port: u16) -> String {
std::env::var("BEACON_ADVERTISE_URL")
.ok()
.map(|s| s.trim().trim_end_matches('/').to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("http://{}:{}", hostname(), listen_port))
}
/// `BEACON_BINDS`, when set.
pub fn binds_from_env() -> Option<String> {
std::env::var("BEACON_BINDS").ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
}
/// Advertise a plugin process: spawn a [`BeaconNode`] named `name` whose single
/// resource is whatever `build` returns, and re-run `build` every interval so a
/// changed manifest (`rev`) is re-advertised at once. Used by both SDK
/// harnesses; returns `None` (and logs) when discovery is disabled or the
/// socket can't be set up — a plugin keeps serving either way.
pub fn advertise_plugin<F>(name: &str, version: &str, build: F) -> Option<BeaconNode>
where
F: Fn() -> Resource + Send + Sync + 'static,
{
if !enabled_from_env() {
tracing::info!("beacon: disabled (ENABLE_UDP_DISCOVERY)");
return None;
}
let cfg = Config::from_env(name).version(version).resource(build());
let interval = cfg.interval;
let node = match BeaconNode::spawn(cfg) {
Ok(n) => n,
Err(e) => {
tracing::warn!(error = %e, "beacon: could not start; plugin will not be discoverable");
return None;
}
};
let refresher = node.clone();
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.tick().await;
loop {
tick.tick().await;
refresher.set_resources(vec![build()]).await;
}
});
Some(node)
}
/// The `/api/neighbors` JSON every service serves (spec Part 2, "HTTP
/// surface"). Each row is a [`SeenNode`] plus `caps` and a v1-compatible
/// `baseUrl` (the `metamesh.service/*` resource's `endpoints.ui`).
///
/// `all` returns every instance instead of one row per name; `cap` keeps only
/// nodes owning a resource that matches it.
pub fn neighbors_json(
node: Option<&BeaconNode>,
current: &str,
all: bool,
cap: Option<&str>,
) -> serde_json::Value {
let Some(node) = node else {
return serde_json::json!({
"current": current, "enabled": false, "count": 0, "neighbors": [],
});
};
let rows: Vec<serde_json::Value> = if all { node.nodes() } else { node.nodes_by_name() }
.into_iter()
.filter(|n| cap.map_or(true, |c| n.matches(c)))
.map(|n| neighbor_row(&n))
.collect();
serde_json::json!({
"current": current,
"enabled": true,
"count": rows.len(),
"self": neighbor_row(&node.self_node()),
"neighbors": rows,
})
}
fn neighbor_row(n: &SeenNode) -> serde_json::Value {
let mut v = serde_json::to_value(n).unwrap_or_default();
if let Some(obj) = v.as_object_mut() {
obj.insert("caps".into(), serde_json::json!(n.caps()));
let base = n.resource(caps::ANY_SERVICE).and_then(|r| r.endpoint("ui"));
if let Some(b) = base {
obj.insert("baseUrl".into(), serde_json::Value::String(b));
}
}
v
}
+719
View File
@@ -0,0 +1,719 @@
//! `BeaconNode` — advertise this process's resources and keep a live view of
//! everyone else's.
//!
//! Ported from meta-discovery v1's `meshdisco.rs`, keeping the two details that
//! fail *silently* when wrong:
//!
//! 1. **Per-interface fan-out.** A socket that joins with `INADDR_ANY` and sends
//! via the default route covers exactly one interface. `metashare-app` sits
//! on `pcs` *and* `metamesh-mesh`; `metawatch-*` on `metawatch` *and*
//! `metamesh-mesh`. We enumerate interfaces and join/send on each, using
//! `ip_mreqn`'s `imr_ifindex`.
//! 2. **Expiry at read time.** No reaper owns correctness; the announce loop's
//! prune only exists to fire change notifications for nodes that went quiet.
//!
//! Raw `libc` rather than `socket2` / `if-addrs`: every consumer already links
//! libc, and this is the whole of what we need from either crate.
use std::collections::HashMap;
use std::io;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde::Serialize;
use tokio::net::UdpSocket;
use tokio::sync::watch;
use tracing::{debug, info, warn};
use super::proto::{
Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT, LIVENESS_FACTOR,
MAX_DATAGRAM, READ_BUFFER, TYPE_ADVERTISE, TYPE_BYE, TYPE_PROBE,
};
/// How a node is set up. Build with [`Config::new`] / [`Config::from_env`].
#[derive(Clone, Debug)]
pub struct Config {
pub node: NodeInfo,
pub resources: Vec<Resource>,
pub group: String,
pub port: u16,
pub interval: Duration,
}
impl Config {
/// Defaults: this host's name as the instance, the standard group/port,
/// a 10 s interval, no resources.
pub fn new(name: impl Into<String>) -> Self {
Self {
node: NodeInfo {
name: name.into(),
instance: hostname(),
version: None,
status: None,
},
resources: Vec::new(),
group: DEFAULT_GROUP.into(),
port: DEFAULT_PORT,
interval: DEFAULT_INTERVAL,
}
}
/// [`Config::new`] with `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS`
/// applied.
pub fn from_env(name: impl Into<String>) -> Self {
let mut c = Self::new(name);
if let Some(g) = env_nonempty("BEACON_GROUP") {
c.group = g;
}
if let Some(p) = env_nonempty("BEACON_PORT").and_then(|v| v.parse().ok()) {
c.port = p;
}
if let Some(ms) = env_nonempty("BEACON_INTERVAL_MS").and_then(|v| v.parse::<u64>().ok()) {
if ms > 0 {
c.interval = Duration::from_millis(ms);
}
}
c
}
pub fn version(mut self, v: impl Into<String>) -> Self {
self.node.version = Some(v.into());
self
}
pub fn instance(mut self, i: impl Into<String>) -> Self {
self.node.instance = i.into();
self
}
pub fn resource(mut self, r: Resource) -> Self {
self.resources.push(r);
self
}
}
/// A node heard on the wire (or this node itself, via [`BeaconNode::self_node`]).
#[derive(Clone, Debug, Serialize)]
pub struct SeenNode {
#[serde(flatten)]
pub node: NodeInfo,
pub resources: Vec<Resource>,
/// Source address taken from the packet, never from the payload.
pub addr: String,
/// Unix seconds.
#[serde(rename = "lastSeen")]
pub last_seen: f64,
}
impl SeenNode {
/// Every cap of every resource, in order, deduplicated.
pub fn caps(&self) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for c in self.resources.iter().flat_map(|r| r.caps.iter()) {
if !out.contains(c) {
out.push(c.clone());
}
}
out
}
/// True if any resource matches `pattern`.
pub fn matches(&self, pattern: &str) -> bool {
self.resources.iter().any(|r| r.matches(pattern))
}
/// The first resource matching `pattern`.
pub fn resource(&self, pattern: &str) -> Option<&Resource> {
self.resources.iter().find(|r| r.matches(pattern))
}
}
/// One resource together with the node advertising it.
#[derive(Clone, Debug, Serialize)]
pub struct SeenResource {
pub node: NodeInfo,
pub resource: Resource,
pub addr: String,
#[serde(rename = "lastSeen")]
pub last_seen: f64,
}
struct Inner {
group: Ipv4Addr,
port: u16,
interval: Duration,
node: RwLock<NodeInfo>,
resources: RwLock<Vec<Resource>>,
socket: UdpSocket,
ifindexes: Vec<(String, u32)>,
seen: Mutex<HashMap<String, SeenNode>>,
changes: watch::Sender<u64>,
stopped: AtomicBool,
}
/// A running beacon v2 participant. Cheap to clone.
#[derive(Clone)]
pub struct BeaconNode {
inner: Arc<Inner>,
}
impl BeaconNode {
/// Bind, join the group on every eligible interface, and start the read and
/// advertise loops. Must be called inside a tokio runtime.
///
/// A join failure on one interface is logged and skipped rather than fatal —
/// a host that blocks multicast must degrade, not fail to boot.
pub fn spawn(cfg: Config) -> io::Result<BeaconNode> {
let group: Ipv4Addr = cfg.group.parse().map_err(|_| {
io::Error::new(io::ErrorKind::InvalidInput, format!("bad group {}", cfg.group))
})?;
let std_sock = bind_reuse(cfg.port)?;
set_multicast_ttl(&std_sock, 1)?;
let ifindexes = eligible_interfaces();
let mut joined = 0usize;
for (name, idx) in &ifindexes {
match join_group(&std_sock, group, *idx) {
Ok(()) => joined += 1,
Err(e) => warn!(iface = %name, error = %e, "beacon: join failed"),
}
}
info!(
group = %cfg.group, port = cfg.port, name = %cfg.node.name,
instance = %cfg.node.instance, resources = cfg.resources.len(),
joined, total = ifindexes.len(),
"beacon: listening"
);
std_sock.set_nonblocking(true)?;
let socket = UdpSocket::from_std(std_sock)?;
let (changes, _) = watch::channel(0u64);
let node = BeaconNode {
inner: Arc::new(Inner {
group,
port: cfg.port,
interval: cfg.interval,
node: RwLock::new(cfg.node),
resources: RwLock::new(cfg.resources),
socket,
ifindexes,
seen: Mutex::new(HashMap::new()),
changes,
stopped: AtomicBool::new(false),
}),
};
let reader = node.clone();
tokio::spawn(async move { reader.read_loop().await });
let advertiser = node.clone();
tokio::spawn(async move {
advertiser.advertise().await;
advertiser.probe(&[]).await;
let mut tick = tokio::time::interval(advertiser.inner.interval);
tick.tick().await; // the immediate first tick
loop {
tick.tick().await;
if advertiser.inner.stopped.load(Ordering::Relaxed) {
break;
}
advertiser.advertise().await;
advertiser.prune();
}
});
Ok(node)
}
/// The instance name this node advertises as.
pub fn instance(&self) -> String {
self.inner.node.read().unwrap().instance.clone()
}
/// Replace this node's resources and re-advertise at once (e.g. a manifest
/// `rev` changed). A no-op if nothing changed.
pub async fn set_resources(&self, resources: Vec<Resource>) {
{
let mut cur = self.inner.resources.write().unwrap();
if *cur == resources {
return;
}
*cur = resources;
}
self.advertise().await;
}
/// Set `node.status` (`starting` / `running`) and re-advertise.
pub async fn set_status(&self, status: &str) {
{
let mut n = self.inner.node.write().unwrap();
if n.status.as_deref() == Some(status) {
return;
}
n.status = Some(status.to_string());
}
self.advertise().await;
}
/// Multicast a probe. Every node owning a resource matching one of `want`
/// (every node, when `want` is empty) replies at once — what makes the
/// protocol usable on a boot path instead of waiting out an interval.
pub async fn probe(&self, want: &[String]) {
let msg = Message::probe(&self.instance(), want);
self.multicast(&msg).await;
}
/// Say goodbye: multicast `bye` and stop advertising. Call on clean
/// shutdown so neighbours drop us at once instead of after the TTL.
pub async fn bye(&self) {
if self.inner.stopped.swap(true, Ordering::Relaxed) {
return;
}
let node = self.inner.node.read().unwrap().clone();
self.multicast(&Message::bye(node)).await;
}
/// Fires (the counter increments) whenever the set of live nodes or any
/// node's resources change — new node, changed advertise, bye, expiry.
/// Consumers re-read [`BeaconNode::resources`] on each change.
pub fn subscribe(&self) -> watch::Receiver<u64> {
self.inner.changes.subscribe()
}
/// Every live node, sorted by (name, instance).
pub fn nodes(&self) -> Vec<SeenNode> {
let cutoff = self.cutoff();
let guard = self.inner.seen.lock().unwrap();
let mut out: Vec<SeenNode> = guard.values().filter(|n| n.last_seen >= cutoff).cloned().collect();
drop(guard);
out.sort_by(|a, b| a.node.name.cmp(&b.node.name).then(a.node.instance.cmp(&b.node.instance)));
out
}
/// One row per node name, most recently seen instance wins — the nav
/// menu's shape.
pub fn nodes_by_name(&self) -> Vec<SeenNode> {
let mut best: HashMap<String, SeenNode> = HashMap::new();
for n in self.nodes() {
match best.get(&n.node.name) {
Some(cur) if cur.last_seen >= n.last_seen => {}
_ => {
best.insert(n.node.name.clone(), n);
}
}
}
let mut out: Vec<SeenNode> = best.into_values().collect();
out.sort_by(|a, b| a.node.name.cmp(&b.node.name));
out
}
/// Every live resource matching `pattern`, with its node.
pub fn resources(&self, pattern: &str) -> Vec<SeenResource> {
self.nodes()
.into_iter()
.flat_map(|n| {
let SeenNode { node, resources, addr, last_seen } = n;
resources.into_iter().filter(|r| r.matches(pattern)).map(move |resource| SeenResource {
node: node.clone(),
resource,
addr: addr.clone(),
last_seen,
})
})
.collect()
}
/// This node as a row, so a UI can render itself without waiting for its
/// own echo.
pub fn self_node(&self) -> SeenNode {
SeenNode {
node: self.inner.node.read().unwrap().clone(),
resources: self.inner.resources.read().unwrap().clone(),
addr: String::new(),
last_seen: now_secs(),
}
}
// -- internals ----------------------------------------------------------
fn cutoff(&self) -> f64 {
now_secs() - self.inner.interval.as_secs_f64() * LIVENESS_FACTOR as f64
}
fn bump(&self) {
self.inner.changes.send_modify(|g| *g = g.wrapping_add(1));
}
/// Drop expired nodes; notify if any went.
fn prune(&self) {
let cutoff = self.cutoff();
let removed = {
let mut guard = self.inner.seen.lock().unwrap();
let before = guard.len();
guard.retain(|_, n| n.last_seen >= cutoff);
before != guard.len()
};
if removed {
self.bump();
}
}
fn advertise_msg(&self) -> Message {
Message::advertise(
self.inner.node.read().unwrap().clone(),
self.inner.resources.read().unwrap().clone(),
)
}
async fn advertise(&self) {
if self.inner.stopped.load(Ordering::Relaxed) {
return;
}
let msg = self.advertise_msg();
self.multicast(&msg).await;
}
fn encode(msg: &Message) -> Option<Vec<u8>> {
match serde_json::to_vec(msg) {
Ok(b) => {
if b.len() > MAX_DATAGRAM {
warn!(bytes = b.len(), max = MAX_DATAGRAM, "beacon: datagram over the fragmentation-safe size");
}
Some(b)
}
Err(e) => {
warn!(error = %e, "beacon: encode failed");
None
}
}
}
/// Writes once per interface. Letting the routing table choose would reach
/// exactly one network — the failure this protocol exists to avoid.
async fn multicast(&self, msg: &Message) {
let Some(body) = Self::encode(msg) else { return };
let dst = SocketAddr::V4(SocketAddrV4::new(self.inner.group, self.inner.port));
for (name, idx) in &self.inner.ifindexes {
if let Err(e) = set_multicast_if(&self.inner.socket, *idx) {
warn!(iface = %name, error = %e, "beacon: select interface failed");
continue;
}
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
warn!(iface = %name, error = %e, "beacon: send failed");
}
}
}
async fn reply_to(&self, dst: SocketAddr) {
let Some(body) = Self::encode(&self.advertise_msg()) else { return };
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
debug!(error = %e, "beacon: reply failed");
}
}
async fn read_loop(self) {
let mut buf = vec![0u8; READ_BUFFER];
loop {
let (n, src) = match self.inner.socket.recv_from(&mut buf).await {
Ok(v) => v,
Err(e) => {
debug!(error = %e, "beacon: read error");
continue;
}
};
let Some(msg) = Message::parse(&buf[..n]) else { continue };
let me = self.instance();
match msg.msg_type.as_str() {
TYPE_PROBE => {
if msg.from.as_deref() == Some(me.as_str()) || self.inner.stopped.load(Ordering::Relaxed) {
continue;
}
let wanted = msg.want.is_empty()
|| self.inner.resources.read().unwrap().iter().any(|r| r.matches_any(&msg.want));
if wanted {
self.reply_to(src).await;
}
}
TYPE_ADVERTISE | TYPE_BYE => {
let Some(node) = msg.node else { continue };
if node.instance == me {
continue; // our own multicast echo
}
let changed = if msg.msg_type == TYPE_BYE {
self.inner.seen.lock().unwrap().remove(&node.instance).is_some()
} else {
let cutoff = self.cutoff();
let mut guard = self.inner.seen.lock().unwrap();
let changed = match guard.get(&node.instance) {
Some(prev) => {
prev.last_seen < cutoff || prev.node != node || prev.resources != msg.resources
}
None => true,
};
guard.insert(
node.instance.clone(),
SeenNode {
node,
resources: msg.resources,
addr: src.ip().to_string(),
last_seen: now_secs(),
},
);
changed
};
if changed {
self.bump();
}
}
_ => {}
}
}
}
}
// ---------------------------------------------------------------------------
// Socket plumbing.
// ---------------------------------------------------------------------------
fn bind_reuse(port: u16) -> io::Result<std::net::UdpSocket> {
unsafe {
let fd: RawFd = libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0);
if fd < 0 {
return Err(io::Error::last_os_error());
}
// Must be set before bind. Lets several listeners share the port on one
// host (a test, a sidecar, a beacon v1 responder in the same netns).
let one: libc::c_int = 1;
for opt in [libc::SO_REUSEADDR, libc::SO_REUSEPORT] {
libc::setsockopt(
fd,
libc::SOL_SOCKET,
opt,
&one as *const _ as *const libc::c_void,
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
);
}
let addr = libc::sockaddr_in {
sin_family: libc::AF_INET as libc::sa_family_t,
sin_port: port.to_be(),
sin_addr: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
sin_zero: [0; 8],
};
if libc::bind(
fd,
&addr as *const _ as *const libc::sockaddr,
std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
) < 0
{
let err = io::Error::last_os_error();
libc::close(fd);
return Err(err);
}
Ok(std::net::UdpSocket::from_raw_fd(fd))
}
}
fn set_multicast_ttl(sock: &std::net::UdpSocket, ttl: libc::c_int) -> io::Result<()> {
// TTL 1: link-local only, never routed off the segment.
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_MULTICAST_TTL,
&ttl as *const _ as *const libc::c_void,
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
/// `ip_mreqn` — `imr_ifindex` is what makes this a per-interface join instead
/// of "whatever the routing table picks".
fn mreqn(group: Ipv4Addr, ifindex: u32) -> libc::ip_mreqn {
libc::ip_mreqn {
imr_multiaddr: libc::in_addr { s_addr: u32::from(group).to_be() },
imr_address: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
imr_ifindex: ifindex as libc::c_int,
}
}
fn join_group(sock: &std::net::UdpSocket, group: Ipv4Addr, ifindex: u32) -> io::Result<()> {
let req = mreqn(group, ifindex);
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_ADD_MEMBERSHIP,
&req as *const _ as *const libc::c_void,
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
fn set_multicast_if(sock: &UdpSocket, ifindex: u32) -> io::Result<()> {
let req = mreqn(Ipv4Addr::UNSPECIFIED, ifindex);
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_MULTICAST_IF,
&req as *const _ as *const libc::c_void,
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
/// Every interface except loopback, as (name, ifindex).
fn eligible_interfaces() -> Vec<(String, u32)> {
let mut out = Vec::new();
unsafe {
let list = libc::if_nameindex();
if list.is_null() {
warn!("beacon: could not enumerate interfaces");
return out;
}
let mut p = list;
while (*p).if_index != 0 && !(*p).if_name.is_null() {
let name = std::ffi::CStr::from_ptr((*p).if_name).to_string_lossy().into_owned();
if name != "lo" {
out.push((name, (*p).if_index));
}
p = p.add(1);
}
libc::if_freenameindex(list);
}
out
}
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
}
/// `$HOSTNAME`, else `/etc/hostname`, else `unknown`.
pub fn hostname() -> String {
env_nonempty("HOSTNAME")
.or_else(|| {
std::fs::read_to_string("/etc/hostname")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
})
.unwrap_or_else(|| "unknown".into())
}
pub(crate) fn now_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs_f64())
.unwrap_or(0.0)
}
/// `ENABLE_UDP_DISCOVERY`, permissive: only `false`/`0`/`no`/`off` disable.
pub fn enabled_from_env() -> bool {
match std::env::var("ENABLE_UDP_DISCOVERY") {
Ok(v) => !matches!(v.trim().to_ascii_lowercase().as_str(), "false" | "0" | "no" | "off"),
Err(_) => true,
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Two nodes on loopback-only hosts can't multicast to each other, so
/// exercise the receive path directly by unicasting into a node's port.
async fn node_on(port: u16, instance: &str, resources: Vec<Resource>) -> BeaconNode {
let mut cfg = Config::new("test").instance(instance);
cfg.port = port;
cfg.interval = Duration::from_millis(200);
cfg.resources = resources;
BeaconNode::spawn(cfg).unwrap()
}
async fn send(port: u16, msg: &Message) {
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(&serde_json::to_vec(msg).unwrap(), ("127.0.0.1", port)).await.unwrap();
}
fn free_port() -> u16 {
std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
}
fn info(instance: &str) -> NodeInfo {
NodeInfo { name: "peer".into(), instance: instance.into(), ..Default::default() }
}
#[tokio::test]
async fn advertise_bye_and_expiry() {
let port = free_port();
let node = node_on(port, "me", vec![]).await;
let mut rx = node.subscribe();
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
send(port, &Message::advertise(info("peer-1"), vec![r])).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert_eq!(node.resources("metamesh.transport/*").len(), 1);
assert!(node.resources("metamesh.core").is_empty());
send(port, &Message::bye(info("peer-1"))).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert!(node.nodes().is_empty());
send(port, &Message::advertise(info("peer-2"), vec![])).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert_eq!(node.nodes().len(), 1);
// 3 × 200 ms liveness, then the prune on the next tick notifies.
tokio::time::timeout(Duration::from_secs(3), rx.changed()).await.unwrap().unwrap();
assert!(node.nodes().is_empty());
}
#[tokio::test]
async fn own_echo_and_v1_ignored() {
let port = free_port();
let node = node_on(port, "me", vec![]).await;
send(port, &Message::advertise(info("me"), vec![])).await;
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(br#"{"type":"announce","name":"x","tools":[]}"#, ("127.0.0.1", port)).await.unwrap();
s.send_to(br#"{"v":1,"type":"announce","name":"meta-sort","instance":"a"}"#, ("127.0.0.1", port))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(150)).await;
assert!(node.nodes().is_empty());
}
#[tokio::test]
async fn probe_reply_honours_want() {
let port = free_port();
let _node = node_on(port, "me", vec![Resource::new("service", ["metamesh.service/meta-sort"])]).await;
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
let mut buf = vec![0u8; READ_BUFFER];
let miss = Message::probe("cli", &["metamesh.core".to_string()]);
s.send_to(&serde_json::to_vec(&miss).unwrap(), ("127.0.0.1", port)).await.unwrap();
assert!(tokio::time::timeout(Duration::from_millis(300), s.recv_from(&mut buf)).await.is_err());
let hit = Message::probe("cli", &["metamesh.service/*".to_string()]);
s.send_to(&serde_json::to_vec(&hit).unwrap(), ("127.0.0.1", port)).await.unwrap();
let (n, _) = tokio::time::timeout(Duration::from_secs(2), s.recv_from(&mut buf)).await.unwrap().unwrap();
let m = Message::parse(&buf[..n]).unwrap();
assert_eq!(m.msg_type, TYPE_ADVERTISE);
assert_eq!(m.node.unwrap().instance, "me");
}
}
+312
View File
@@ -0,0 +1,312 @@
//! Beacon v2 wire types and capability matching.
//!
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`.
use std::collections::BTreeMap;
use std::hash::{Hash, Hasher};
use std::time::Duration;
use serde::{Deserialize, Serialize};
pub const PROTO: &str = "beacon";
pub const VERSION: u8 = 2;
/// Beacon v1's group and port, on purpose — one well-known endpoint for every
/// kind of local resource. v1 and v2 coexist because v2 never sends v1's
/// `discovery` / `announce` types.
pub const DEFAULT_GROUP: &str = "239.255.99.1";
pub const DEFAULT_PORT: u16 = 9099;
pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(10);
/// A node unheard for `interval * this` is dropped (evaluated at read time).
pub const LIVENESS_FACTOR: u32 = 3;
/// Receive buffer. Senders stay under [`MAX_DATAGRAM`].
pub const READ_BUFFER: usize = 8192;
/// Above this a datagram risks IP fragmentation; the sender logs a warning.
pub const MAX_DATAGRAM: usize = 1400;
pub const TYPE_PROBE: &str = "probe";
pub const TYPE_ADVERTISE: &str = "advertise";
pub const TYPE_BYE: &str = "bye";
/// Who is advertising.
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct NodeInfo {
/// What the software is (`meta-sort`, `meta-transport-nzb`).
pub name: String,
/// Unique per running copy — the container hostname.
pub instance: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
/// `starting` | `running`. Absent reads as `running`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub status: Option<String>,
}
/// One advertised resource.
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct Resource {
/// Unique within the node.
pub id: String,
pub caps: Vec<String>,
/// Well-known names: `http`, `ui`, `manifest`, `mcp`. A value starting with
/// `/` is relative to `http` — see [`Resource::endpoint`].
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub endpoints: BTreeMap<String, String>,
/// Opaque revision of the resource's details (e.g. its manifest).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rev: Option<String>,
/// The one consumer instance this resource belongs to. Absent = anyone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub binds: Option<String>,
/// Profile-defined, per capability.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data: Option<serde_json::Value>,
}
impl Resource {
pub fn new(id: impl Into<String>, caps: impl IntoIterator<Item = impl Into<String>>) -> Self {
Self {
id: id.into(),
caps: caps.into_iter().map(Into::into).collect(),
..Default::default()
}
}
pub fn with_endpoint(mut self, name: impl Into<String>, url: impl Into<String>) -> Self {
let url = url.into();
if !url.is_empty() {
self.endpoints.insert(name.into(), url);
}
self
}
/// True if any of this resource's caps matches `pattern`.
pub fn matches(&self, pattern: &str) -> bool {
self.caps.iter().any(|c| cap_matches(pattern, c))
}
/// True if any cap matches any of `patterns`. An empty list matches.
pub fn matches_any(&self, patterns: &[String]) -> bool {
patterns.is_empty() || patterns.iter().any(|p| self.matches(p))
}
/// `binds` is absent, or names `instance`.
pub fn usable_by(&self, instance: &str) -> bool {
self.binds.as_deref().map_or(true, |b| b == instance)
}
/// The named endpoint as an absolute URL: a value starting with `/` is
/// joined onto `endpoints.http`.
pub fn endpoint(&self, name: &str) -> Option<String> {
let v = self.endpoints.get(name)?;
if v.starts_with('/') {
let base = self.endpoints.get("http")?;
Some(format!("{}{}", base.trim_end_matches('/'), v))
} else {
Some(v.clone())
}
}
}
/// Every v2 datagram.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Message {
pub proto: String,
pub v: u8,
#[serde(rename = "type")]
pub msg_type: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub node: Option<NodeInfo>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub resources: Vec<Resource>,
/// Probe only: informational sender instance.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub from: Option<String>,
/// Probe only: reply only if a resource matches one of these.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub want: Vec<String>,
}
impl Message {
fn base(msg_type: &str) -> Self {
Self {
proto: PROTO.into(),
v: VERSION,
msg_type: msg_type.into(),
node: None,
resources: Vec::new(),
from: None,
want: Vec::new(),
}
}
pub fn probe(from: &str, want: &[String]) -> Self {
Self {
from: Some(from.to_string()),
want: want.to_vec(),
..Self::base(TYPE_PROBE)
}
}
pub fn advertise(node: NodeInfo, resources: Vec<Resource>) -> Self {
Self {
node: Some(node),
resources,
..Self::base(TYPE_ADVERTISE)
}
}
pub fn bye(node: NodeInfo) -> Self {
Self {
node: Some(node),
..Self::base(TYPE_BYE)
}
}
/// Decode a datagram, returning `None` for anything that is not a
/// well-formed beacon v2 message — beacon v1 traffic (no envelope), other
/// versions, unknown types, or an advertise/bye without a node. The group
/// is shared, so all of that is silently ignored.
pub fn parse(bytes: &[u8]) -> Option<Self> {
let m: Message = serde_json::from_slice(bytes).ok()?;
if m.proto != PROTO || m.v != VERSION {
return None;
}
match m.msg_type.as_str() {
TYPE_PROBE => Some(m),
TYPE_ADVERTISE | TYPE_BYE => {
let n = m.node.as_ref()?;
if n.instance.is_empty() || n.name.is_empty() {
return None;
}
Some(m)
}
_ => None,
}
}
}
/// Split `base@contract`. A non-numeric or missing contract is `None`.
fn split_contract(s: &str) -> (&str, Option<&str>) {
match s.rsplit_once('@') {
Some((base, c)) => (base, Some(c)),
None => (s, None),
}
}
/// Does capability `cap` satisfy `pattern`?
///
/// - `*` matches everything.
/// - `@N` on the pattern must equal the cap's contract; without one the cap's
/// contract is ignored.
/// - `x/*` matches any variant of `x`; otherwise the bases must be equal.
pub fn cap_matches(pattern: &str, cap: &str) -> bool {
if pattern == "*" {
return true;
}
let (pbase, pcontract) = split_contract(pattern);
let (cbase, ccontract) = split_contract(cap);
if let Some(pc) = pcontract {
if ccontract != Some(pc) {
return false;
}
}
match pbase.strip_suffix("/*") {
Some(prefix) => cbase
.strip_prefix(prefix)
.is_some_and(|rest| rest.starts_with('/') && rest.len() > 1),
None => pbase == cbase,
}
}
/// A short opaque revision for any serialisable value — what a harness puts in
/// [`Resource::rev`] so consumers re-fetch a manifest only when it changed.
/// Stable for a given build; not cryptographic and not meant to be.
pub fn rev_of<T: Serialize>(value: &T) -> String {
let bytes = serde_json::to_vec(value).unwrap_or_default();
let mut h = std::collections::hash_map::DefaultHasher::new();
bytes.hash(&mut h);
format!("{:016x}", h.finish())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn wildcard_matches_any_variant_only() {
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/nzb@1"));
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/ipfs"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transportx/a"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport/"));
assert!(cap_matches("*", "anything@3"));
}
#[test]
fn contract_rules() {
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@1"));
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@2"));
assert!(cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@2"));
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@1"));
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb"));
assert!(cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@1"));
assert!(!cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@2"));
}
#[test]
fn exact_match() {
assert!(cap_matches("mcp", "mcp"));
assert!(!cap_matches("mcp", "mcp/x"));
assert!(!cap_matches("metamesh.core", "metamesh.core2"));
assert!(!cap_matches("metamesh.transport/nzb", "metamesh.transport/ipfs@1"));
}
#[test]
fn parse_rejects_v1_and_foreign_traffic() {
// beacon v1 aggregator probe / server announce
assert!(Message::parse(br#"{"type":"discovery"}"#).is_none());
assert!(Message::parse(br#"{"type":"announce","name":"x","tools":[]}"#).is_none());
// meta-discovery v1
assert!(Message::parse(br#"{"v":1,"type":"announce","name":"meta-sort"}"#).is_none());
// wrong version / unknown type / not json
assert!(Message::parse(br#"{"proto":"beacon","v":3,"type":"probe"}"#).is_none());
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"announce"}"#).is_none());
assert!(Message::parse(b"\x00\x01garbage").is_none());
// advertise without a node
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"advertise"}"#).is_none());
}
#[test]
fn roundtrip_advertise() {
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"])
.with_endpoint("http", "http://metashare-nzb:3000")
.with_endpoint("manifest", "/manifest");
let m = Message::advertise(
NodeInfo { name: "meta-transport-nzb".into(), instance: "metashare-nzb".into(), ..Default::default() },
vec![r],
);
let bytes = serde_json::to_vec(&m).unwrap();
let back = Message::parse(&bytes).unwrap();
assert_eq!(back.msg_type, TYPE_ADVERTISE);
let r = &back.resources[0];
assert_eq!(r.endpoint("manifest").as_deref(), Some("http://metashare-nzb:3000/manifest"));
assert!(r.matches("metamesh.transport/*"));
}
#[test]
fn binds() {
let mut r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
assert!(r.usable_by("anyone"));
r.binds = Some("metashare-app".into());
assert!(r.usable_by("metashare-app"));
assert!(!r.usable_by("metawatch-share"));
}
#[test]
fn probe_filter() {
let r = Resource::new("service", ["metamesh.service/meta-sort"]);
assert!(r.matches_any(&[]));
assert!(r.matches_any(&["metamesh.service/*".into()]));
assert!(!r.matches_any(&["metamesh.core".into()]));
}
}
+234
View File
@@ -0,0 +1,234 @@
//! On-demand discovery scan — what a consumer's "Scan" button runs.
//!
//! Probe everyone, wait a moment for the replies, then classify every resource
//! matching the consumer's capability pattern against what the consumer has
//! already configured. The report is the same JSON in every consumer
//! (meta-share, meta-gateway, meta-sort), rendered by the shared
//! `<meta-beacon-scan>` element.
use std::time::{Duration, Instant};
use serde::Serialize;
use super::node::{BeaconNode, SeenResource};
/// How a discovered resource relates to this consumer.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum CandidateState {
/// Its URL is already in the consumer's list.
Configured,
/// Can be added.
Addable,
/// `binds` names another consumer; shown but not addable.
BoundElsewhere,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Candidate {
pub instance: String,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
pub resource_id: String,
pub caps: Vec<String>,
/// `endpoints.http` — what the consumer would register.
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub binds: Option<String>,
pub state: CandidateState,
/// The list entry that already holds this URL (state `configured`).
#[serde(skip_serializing_if = "Option::is_none")]
pub configured_as: Option<String>,
/// A list-safe name to register it under.
pub suggested_name: String,
}
#[derive(Clone, Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ScanSummary {
/// Every node heard on the segment, whatever it advertises.
pub nodes: usize,
/// Resources matching the consumer's pattern.
pub capable: usize,
pub configured: usize,
pub bound_elsewhere: usize,
pub addable: usize,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ScanReport {
pub cap: String,
/// This consumer's instance (what `binds` is compared against).
#[serde(rename = "self")]
pub self_instance: String,
pub duration_ms: u64,
pub summary: ScanSummary,
pub candidates: Vec<Candidate>,
}
fn norm(u: &str) -> String {
u.trim().trim_end_matches('/').to_string()
}
/// Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules.
pub fn suggested_name(instance: &str) -> String {
let s: String = instance
.trim()
.to_ascii_lowercase()
.chars()
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '-' })
.collect();
if s.is_empty() { "plugin".into() } else { s }
}
/// Pure classification, for tests and for callers that already hold a view.
/// `configured` is `(name, url)` of every entry the consumer has.
pub fn classify(
pattern: &str,
self_instance: &str,
nodes: usize,
resources: Vec<SeenResource>,
configured: &[(String, String)],
) -> ScanReport {
let mut summary = ScanSummary { nodes, ..Default::default() };
let mut candidates = Vec::new();
for r in resources {
// A resource without an http endpoint cannot be registered anywhere.
let Some(url) = r.resource.endpoint("http") else { continue };
let url = norm(&url);
summary.capable += 1;
let configured_as = configured.iter().find(|(_, u)| norm(u) == url).map(|(n, _)| n.clone());
let state = if configured_as.is_some() {
summary.configured += 1;
CandidateState::Configured
} else if !r.resource.usable_by(self_instance) {
summary.bound_elsewhere += 1;
CandidateState::BoundElsewhere
} else {
summary.addable += 1;
CandidateState::Addable
};
candidates.push(Candidate {
suggested_name: suggested_name(&r.node.instance),
instance: r.node.instance,
name: r.node.name,
version: r.node.version,
resource_id: r.resource.id,
caps: r.resource.caps,
url,
binds: r.resource.binds,
state,
configured_as,
});
}
// Addable first (what the operator came for), then configured, then the rest.
let rank = |s: CandidateState| match s {
CandidateState::Addable => 0,
CandidateState::Configured => 1,
CandidateState::BoundElsewhere => 2,
};
candidates.sort_by(|a, b| rank(a.state).cmp(&rank(b.state)).then(a.instance.cmp(&b.instance)));
ScanReport {
cap: pattern.to_string(),
self_instance: self_instance.to_string(),
duration_ms: 0,
summary,
candidates,
}
}
/// Probe every node, wait `wait` for the replies, then classify what matches
/// `pattern`. A node that stays silent through the wait but is still inside
/// its liveness window is counted too — the view is "who is here", not "who
/// answered this probe".
pub async fn scan(
node: &BeaconNode,
pattern: &str,
configured: &[(String, String)],
wait: Duration,
) -> ScanReport {
let started = Instant::now();
node.probe(&[]).await;
tokio::time::sleep(wait).await;
let mut report = classify(
pattern,
&node.instance(),
node.nodes().len(),
node.resources(pattern),
configured,
);
report.duration_ms = started.elapsed().as_millis() as u64;
report
}
#[cfg(test)]
mod tests {
use super::*;
use crate::beacon::{NodeInfo, Resource};
fn seen(instance: &str, cap: &str, url: Option<&str>, binds: Option<&str>) -> SeenResource {
let mut r = Resource::new("x", [cap]);
if let Some(u) = url {
r = r.with_endpoint("http", u);
}
r.binds = binds.map(Into::into);
SeenResource {
node: NodeInfo { name: "plugin".into(), instance: instance.into(), ..Default::default() },
resource: r,
addr: "10.0.0.1".into(),
last_seen: 0.0,
}
}
#[test]
fn classifies_configured_addable_and_bound_elsewhere() {
let resources = vec![
seen("metashare-nzb", "metamesh.transport/nzb@1", Some("http://metashare-nzb:3000/"), Some("metashare-app")),
seen("qbit", "metamesh.transport/torrent@1", Some("http://qbit:3000"), None),
seen("other-ipfs", "metamesh.transport/ipfs@1", Some("http://other-ipfs:3000"), Some("metawatch-share")),
seen("no-endpoint", "metamesh.transport/ipfs@1", None, None),
];
let configured = vec![("nzb".to_string(), "http://metashare-nzb:3000".to_string())];
let r = classify("metamesh.transport/*", "metashare-app", 10, resources, &configured);
assert_eq!(r.summary.nodes, 10);
assert_eq!(r.summary.capable, 3, "a resource without an http endpoint is not a candidate");
assert_eq!((r.summary.configured, r.summary.addable, r.summary.bound_elsewhere), (1, 1, 1));
assert_eq!(r.candidates[0].instance, "qbit", "addable rows sort first");
assert_eq!(r.candidates[1].configured_as.as_deref(), Some("nzb"));
assert_eq!(r.candidates[2].state, CandidateState::BoundElsewhere);
let json = serde_json::to_value(&r).unwrap();
assert_eq!(json["self"], "metashare-app");
assert_eq!(json["summary"]["boundElsewhere"], 1);
assert_eq!(json["candidates"][2]["state"], "bound-elsewhere");
}
#[test]
fn suggested_names_are_list_safe() {
assert_eq!(suggested_name("MetaShare NZB"), "metashare-nzb");
assert_eq!(suggested_name(""), "plugin");
}
#[tokio::test]
async fn scan_reports_what_the_node_heard() {
use crate::beacon::{BeaconNode, Config, Message};
let port = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
let mut cfg = Config::new("consumer").instance("me");
cfg.port = port;
let node = BeaconNode::spawn(cfg).unwrap();
let ad = Message::advertise(
NodeInfo { name: "meta-feeder-tmdb".into(), instance: "tmdb-feeder".into(), ..Default::default() },
vec![Resource::new("feeder", ["metamesh.feeder/tmdb"]).with_endpoint("http", "http://tmdb-feeder:8080")],
);
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(&serde_json::to_vec(&ad).unwrap(), ("127.0.0.1", port)).await.unwrap();
let r = scan(&node, "metamesh.feeder/*", &[], Duration::from_millis(200)).await;
assert_eq!(r.summary.nodes, 1);
assert_eq!(r.summary.addable, 1);
assert_eq!(r.candidates[0].url, "http://tmdb-feeder:8080");
assert_eq!(r.candidates[0].suggested_name, "tmdb-feeder");
assert!(r.duration_ms >= 200);
}
}
+602
View File
@@ -0,0 +1,602 @@
//! MMUR2 signed claims: signer `uid` says `key = value` about content `cid`, at
//! `ts` (milliseconds, also the version). See meta-root
//! `docs/implementation/metacore-claims-model.md` §3.
//!
//! Wire spec (pinned by `tests/claim-vectors.json`, vendored from meta-core's
//! `internal/claim/testdata` — `scripts/check-claim-vectors.sh` guards it):
//!
//! ```text
//! domain = b"MMUR2\n"
//! payload = canonical CBOR array [cid, uid, key, value, ts]
//! bytes_to_sign = domain || payload
//! sig = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S
//! ```
//!
//! `value` is CBOR text (public), bytes (private envelope) or null (retraction).
//! The encoder is hand-rolled: those five shapes are all MMUR2 needs, and a
//! hand-rolled head encoder cannot drift from "smallest integer form".
//!
//! The type and its JSON form are always available with the `feeder` feature
//! (the gateway wire carries claims); signing and verification are here too,
//! behind `claims`, which `feeder` enables.
use std::collections::BTreeMap;
use base64::Engine as _;
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
/// Domain separator. Changing it invalidates every claim everywhere.
pub const DOMAIN: &[u8] = b"MMUR2\n";
/// A claim's value. The SQLite storage type in meta-core mirrors it.
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub enum ClaimValue {
/// Retraction (tombstone).
#[default]
Null,
/// Public plaintext.
Text(String),
/// Private envelope (XChaCha20-Poly1305), opaque to everyone but the owner.
Bytes(Vec<u8>),
}
impl ClaimValue {
pub fn is_null(&self) -> bool {
matches!(self, ClaimValue::Null)
}
pub fn as_text(&self) -> Option<&str> {
match self {
ClaimValue::Text(s) => Some(s),
_ => None,
}
}
}
/// One signed vote.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SignedClaim {
pub cid: String,
pub uid: String,
pub key: String,
#[serde(with = "value_json")]
pub value: ClaimValue,
pub ts: u64,
/// 64-byte compact r||s, base64 in JSON.
#[serde(with = "sig_b64")]
pub sig: Vec<u8>,
}
impl SignedClaim {
/// The exact bytes the signature covers.
pub fn signed_bytes(&self) -> Vec<u8> {
signed_bytes(&self.cid, &self.uid, &self.key, &self.value, self.ts)
}
}
/// `domain || canonical CBOR [cid, uid, key, value, ts]`.
pub fn signed_bytes(cid: &str, uid: &str, key: &str, value: &ClaimValue, ts: u64) -> Vec<u8> {
let extra = match value {
ClaimValue::Text(s) => s.len(),
ClaimValue::Bytes(b) => b.len(),
ClaimValue::Null => 0,
};
let mut b = Vec::with_capacity(DOMAIN.len() + cid.len() + uid.len() + key.len() + extra + 32);
b.extend_from_slice(DOMAIN);
head(&mut b, 4, 5);
text(&mut b, cid);
text(&mut b, uid);
text(&mut b, key);
match value {
ClaimValue::Text(s) => text(&mut b, s),
ClaimValue::Bytes(bytes) => {
head(&mut b, 2, bytes.len() as u64);
b.extend_from_slice(bytes);
}
ClaimValue::Null => b.push(0xf6),
}
head(&mut b, 0, ts);
b
}
fn text(b: &mut Vec<u8>, s: &str) {
head(b, 3, s.len() as u64);
b.extend_from_slice(s.as_bytes());
}
/// CBOR major type + argument, smallest form.
fn head(b: &mut Vec<u8>, major: u8, n: u64) {
let m = major << 5;
if n < 24 {
b.push(m | n as u8);
} else if n <= 0xff {
b.extend_from_slice(&[m | 24, n as u8]);
} else if n <= 0xffff {
b.push(m | 25);
b.extend_from_slice(&(n as u16).to_be_bytes());
} else if n <= 0xffff_ffff {
b.push(m | 26);
b.extend_from_slice(&(n as u32).to_be_bytes());
} else {
b.push(m | 27);
b.extend_from_slice(&n.to_be_bytes());
}
}
/// JSON (human-readable) uses the spec §3.4 form — a string, `{"b64": …}` or
/// `null`. Binary formats (the bincode gateway wire) use a plain tagged enum:
/// bincode cannot decode the untagged JSON shape.
mod value_json {
use super::*;
#[derive(Serialize, Deserialize)]
struct B64 {
b64: String,
}
#[derive(Serialize, Deserialize)]
enum Wire {
Null,
Text(String),
Bytes(Vec<u8>),
}
pub fn serialize<S: Serializer>(v: &ClaimValue, s: S) -> Result<S::Ok, S::Error> {
if !s.is_human_readable() {
let w = match v {
ClaimValue::Null => Wire::Null,
ClaimValue::Text(t) => Wire::Text(t.clone()),
ClaimValue::Bytes(b) => Wire::Bytes(b.clone()),
};
return w.serialize(s);
}
match v {
ClaimValue::Null => s.serialize_none(),
ClaimValue::Text(t) => s.serialize_str(t),
ClaimValue::Bytes(b) => B64 {
b64: base64::engine::general_purpose::STANDARD.encode(b),
}
.serialize(s),
}
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<ClaimValue, D::Error> {
if !d.is_human_readable() {
return Ok(match Wire::deserialize(d)? {
Wire::Null => ClaimValue::Null,
Wire::Text(t) => ClaimValue::Text(t),
Wire::Bytes(b) => ClaimValue::Bytes(b),
});
}
#[derive(Deserialize)]
#[serde(untagged)]
enum Raw {
Text(String),
Bytes(B64),
}
Ok(match Option::<Raw>::deserialize(d)? {
None => ClaimValue::Null,
Some(Raw::Text(t)) => ClaimValue::Text(t),
Some(Raw::Bytes(B64 { b64 })) => ClaimValue::Bytes(
base64::engine::general_purpose::STANDARD
.decode(b64)
.map_err(de::Error::custom)?,
),
})
}
}
/// base64 in JSON, raw bytes in binary formats.
mod sig_b64 {
use super::*;
pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
if !s.is_human_readable() {
// A plain sequence, symmetric with `Vec::<u8>::deserialize` below.
return v.serialize(s);
}
s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(v))
}
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
if !d.is_human_readable() {
return Vec::<u8>::deserialize(d);
}
let s = String::deserialize(d)?;
base64::engine::general_purpose::STANDARD
.decode(s)
.map_err(de::Error::custom)
}
}
/// Union-flatten a claim list into a field map: every key that has at least
/// one non-retracted text vote, with the first such value seen. A
/// **superset view for filtering and scoring**, not a merge — the merge (most
/// signers → oldest ts → smallest value, own vote first) lives only in
/// meta-core (`POST /api/merge`). Ciphertext never appears.
pub fn union_fields(claims: &[SignedClaim]) -> BTreeMap<String, String> {
let mut out = BTreeMap::new();
for c in claims {
if let ClaimValue::Text(v) = &c.value {
out.entry(c.key.clone()).or_insert_with(|| v.clone());
}
}
out
}
#[cfg(feature = "claims")]
pub use crypto::*;
#[cfg(feature = "claims")]
mod crypto {
use super::*;
use k256::ecdsa::signature::hazmat::{PrehashSigner, PrehashVerifier};
use k256::ecdsa::{Signature, SigningKey, VerifyingKey};
use sha2::{Digest, Sha256};
/// Why a claim failed verification.
#[derive(Debug, thiserror::Error)]
pub enum ClaimError {
#[error("claim: missing cid or key")]
Shape,
#[error("uid: {0}")]
Uid(String),
#[error("sig: want a 64-byte compact signature")]
SigLen,
#[error("sig: high-S signatures are refused (MMUR2 is low-S only)")]
HighS,
#[error("sig: does not verify")]
Invalid,
}
/// Recover the verifying key a uid encodes (`"z" + base58btc(33-byte sec1)`).
pub fn pubkey_from_uid(uid: &str) -> Result<VerifyingKey, ClaimError> {
let body = uid
.strip_prefix('z')
.ok_or_else(|| ClaimError::Uid("missing multibase 'z' prefix".into()))?;
let raw = bs58::decode(body)
.into_vec()
.map_err(|e| ClaimError::Uid(e.to_string()))?;
if raw.len() != 33 {
return Err(ClaimError::Uid(format!(
"pubkey is {} bytes, want 33",
raw.len()
)));
}
VerifyingKey::from_sec1_bytes(&raw).map_err(|e| ClaimError::Uid(e.to_string()))
}
/// The uid of a verifying key.
pub fn uid_from_pubkey(vk: &VerifyingKey) -> String {
let point = vk.to_encoded_point(true);
format!("z{}", bs58::encode(point.as_bytes()).into_string())
}
impl SignedClaim {
/// Check shape, low-S and the signature against the uid's key.
pub fn verify(&self) -> Result<(), ClaimError> {
if self.cid.is_empty() || self.key.is_empty() {
return Err(ClaimError::Shape);
}
if self.sig.len() != 64 {
return Err(ClaimError::SigLen);
}
let sig = Signature::from_slice(&self.sig).map_err(|_| ClaimError::Invalid)?;
if sig.normalize_s().is_some() {
return Err(ClaimError::HighS);
}
let vk = pubkey_from_uid(&self.uid)?;
let digest = Sha256::digest(self.signed_bytes());
vk.verify_prehash(&digest, &sig)
.map_err(|_| ClaimError::Invalid)
}
}
/// A writer's identity: one key per writing container (spec D2).
#[derive(Clone)]
pub struct Signer {
key: SigningKey,
uid: String,
}
impl std::fmt::Debug for Signer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Signer")
.field("uid", &self.uid)
.finish_non_exhaustive()
}
}
impl Signer {
/// From a 32-byte secret, hex (optionally `0x`-prefixed).
pub fn from_hex(hex_key: &str) -> anyhow::Result<Self> {
let s = hex_key.trim();
let s = s
.strip_prefix("0x")
.or_else(|| s.strip_prefix("0X"))
.unwrap_or(s);
let raw = decode_hex(s).ok_or_else(|| anyhow::anyhow!("signing key is not hex"))?;
if raw.len() != 32 {
anyhow::bail!("signing key must be 32 bytes, got {}", raw.len());
}
let key = SigningKey::from_slice(&raw)
.map_err(|e| anyhow::anyhow!("invalid signing key: {e}"))?;
let uid = uid_from_pubkey(key.verifying_key());
Ok(Signer { key, uid })
}
/// A fresh random key.
pub fn generate() -> Self {
let key = SigningKey::random(&mut rand_core::OsRng);
let uid = uid_from_pubkey(key.verifying_key());
Signer { key, uid }
}
pub fn uid(&self) -> &str {
&self.uid
}
/// The secret as lowercase hex (for writing a generated key file).
pub fn secret_hex(&self) -> String {
self.key
.to_bytes()
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
/// Sign one claim at `ts`. RFC 6979 deterministic, normalised to low-S.
pub fn sign(&self, cid: &str, key: &str, value: ClaimValue, ts: u64) -> SignedClaim {
let bytes = signed_bytes(cid, &self.uid, key, &value, ts);
let digest = Sha256::digest(&bytes);
let sig: Signature = self
.key
.sign_prehash(&digest)
.expect("a 32-byte prehash always signs");
let sig = sig.normalize_s().unwrap_or(sig);
SignedClaim {
cid: cid.to_string(),
uid: self.uid.clone(),
key: key.to_string(),
value,
ts,
sig: sig.to_bytes().to_vec(),
}
}
/// Sign every field of a flat map as text claims on `cid`, all at `ts`.
pub fn sign_fields(
&self,
cid: &str,
fields: &BTreeMap<String, String>,
ts: u64,
) -> Vec<SignedClaim> {
fields
.iter()
.map(|(k, v)| self.sign(cid, k, ClaimValue::Text(v.clone()), ts))
.collect()
}
/// A signed retraction of `key` on `cid`.
pub fn retract(&self, cid: &str, key: &str, ts: u64) -> SignedClaim {
self.sign(cid, key, ClaimValue::Null, ts)
}
}
/// Milliseconds since the epoch — the `ts` of a claim signed now. Strictly
/// increasing per process: `ts` is the version, so two writes to one key in
/// the same millisecond must not make the second one `stale`.
pub fn now_ms() -> u64 {
use std::sync::atomic::{AtomicU64, Ordering};
static LAST: AtomicU64 = AtomicU64::new(0);
let wall = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0);
let mut last = LAST.load(Ordering::Relaxed);
loop {
let next = wall.max(last + 1);
match LAST.compare_exchange_weak(last, next, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => return next,
Err(seen) => last = seen,
}
}
}
/// Env var holding the signer's secret as hex.
pub const SIGNING_KEY_ENV: &str = "METAMESH_SIGNING_KEY";
/// Env var naming a key file (bind-mounted); generated on first boot.
pub const SIGNING_KEY_FILE_ENV: &str = "METAMESH_SIGNING_KEY_FILE";
/// The writer's key (spec §9): `METAMESH_SIGNING_KEY` (hex) wins; else
/// `METAMESH_SIGNING_KEY_FILE`, generated and written (mode 0600) when the
/// file does not exist yet. Neither set → error: a writer must never sign
/// with a silent ephemeral identity.
pub fn load_or_create_key() -> anyhow::Result<Signer> {
if let Some(hex) = env_nonempty(SIGNING_KEY_ENV) {
return Signer::from_hex(&hex);
}
let Some(path) = env_nonempty(SIGNING_KEY_FILE_ENV) else {
anyhow::bail!("no signing key: set {SIGNING_KEY_ENV} or {SIGNING_KEY_FILE_ENV}");
};
load_or_create_key_file(std::path::Path::new(&path))
}
/// Read a hex key file, or generate one there (mode 0600) if absent.
pub fn load_or_create_key_file(path: &std::path::Path) -> anyhow::Result<Signer> {
match std::fs::read_to_string(path) {
Ok(s) => Signer::from_hex(&s),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
let signer = Signer::generate();
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
write_secret(path, &signer.secret_hex())?;
tracing::info!(target: "meta-feeder::claims", uid = %signer.uid(),
path = %path.display(), "generated a new signing key");
Ok(signer)
}
Err(e) => Err(anyhow::anyhow!("read signing key {}: {e}", path.display())),
}
}
#[cfg(unix)]
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
use std::io::Write;
use std::os::unix::fs::OpenOptionsExt;
let mut f = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)?;
f.write_all(hex.as_bytes())?;
f.write_all(b"\n")
}
#[cfg(not(unix))]
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
std::fs::write(path, format!("{hex}\n"))
}
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key)
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
}
fn decode_hex(s: &str) -> Option<Vec<u8>> {
if s.len() % 2 != 0 {
return None;
}
(0..s.len())
.step_by(2)
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Deserialize)]
struct Vector {
name: String,
cid: String,
uid: String,
key: String,
value: Option<String>,
#[serde(rename = "valueBytesHex")]
value_bytes_hex: Option<String>,
ts: u64,
#[serde(rename = "signedBytesHex")]
signed: String,
#[serde(rename = "sigHex")]
sig: String,
}
#[derive(Deserialize)]
struct HighS {
vector: String,
#[serde(rename = "sigHex")]
sig: String,
}
#[derive(Deserialize)]
struct File {
#[serde(rename = "privateKeyHex")]
private_key_hex: String,
uid: String,
vectors: Vec<Vector>,
#[serde(rename = "highS")]
high_s: HighS,
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn load() -> File {
serde_json::from_str(include_str!("../tests/claim-vectors.json")).unwrap()
}
fn value(v: &Vector) -> ClaimValue {
match (&v.value_bytes_hex, &v.value) {
(Some(h), _) => ClaimValue::Bytes(decode_hex(h).unwrap()),
(None, Some(t)) => ClaimValue::Text(t.clone()),
(None, None) => ClaimValue::Null,
}
}
#[test]
fn golden_vectors() {
let f = load();
let signer = Signer::from_hex(&f.private_key_hex).unwrap();
assert_eq!(signer.uid(), f.uid);
for v in &f.vectors {
let c = signer.sign(&v.cid, &v.key, value(v), v.ts);
assert_eq!(c.uid, v.uid, "{}", v.name);
assert_eq!(hex(&c.signed_bytes()), v.signed, "{}: signed bytes", v.name);
assert_eq!(hex(&c.sig), v.sig, "{}: RFC 6979 signature", v.name);
c.verify().unwrap_or_else(|e| panic!("{}: {e}", v.name));
// JSON round trip keeps the claim verifiable.
let back: SignedClaim =
serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap();
assert_eq!(back, c, "{}: json", v.name);
}
let v = f
.vectors
.iter()
.find(|v| v.name == f.high_s.vector)
.unwrap();
let high = SignedClaim {
cid: v.cid.clone(),
uid: v.uid.clone(),
key: v.key.clone(),
value: value(v),
ts: v.ts,
sig: decode_hex(&f.high_s.sig).unwrap(),
};
assert!(matches!(high.verify(), Err(ClaimError::HighS)));
}
#[test]
fn tamper_is_refused() {
let s = Signer::generate();
let mut c = s.sign("bafy", "title", ClaimValue::Text("A".into()), 1);
c.value = ClaimValue::Text("B".into());
assert!(matches!(c.verify(), Err(ClaimError::Invalid)));
}
#[test]
fn key_file_is_generated_once() {
let dir = tempfile::tempdir().unwrap();
let p = dir.path().join("keys/signing.key");
let a = load_or_create_key_file(&p).unwrap();
let b = load_or_create_key_file(&p).unwrap();
assert_eq!(a.uid(), b.uid());
}
#[test]
fn json_shapes() {
let s = Signer::generate();
let j =
serde_json::to_value(s.sign("c", "k", ClaimValue::Bytes(vec![1, 2]), 5)).unwrap();
assert_eq!(j["value"]["b64"], "AQI=");
let j = serde_json::to_value(s.retract("c", "k", 6)).unwrap();
assert!(j["value"].is_null());
}
#[test]
fn union_keeps_first_text_and_skips_retractions_and_cipher() {
let s = Signer::generate();
let claims = vec![
s.sign("c", "title", ClaimValue::Text("A".into()), 1),
s.sign("c", "title", ClaimValue::Text("B".into()), 2),
s.retract("c", "gone", 3),
s.sign("c", "seek", ClaimValue::Bytes(vec![9]), 4),
];
let f = union_fields(&claims);
assert_eq!(f.get("title").map(String::as_str), Some("A"));
assert!(!f.contains_key("gone") && !f.contains_key("seek"));
}
}
}
+31 -7
View File
@@ -64,6 +64,10 @@ pub struct EnrichmentConfig {
pub opensubtitles_password: Option<String>, pub opensubtitles_password: Option<String>,
/// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
pub opensubtitles_languages: String, pub opensubtitles_languages: String,
/// This feeder's claim signer (`METAMESH_SIGNING_KEY` / `_FILE`). `None`
/// when no key is configured: every meta-core write then fails loudly
/// instead of publishing unsigned data.
pub signer: Option<crate::claims::Signer>,
} }
fn env_nonempty(key: &str) -> Option<String> { fn env_nonempty(key: &str) -> Option<String> {
@@ -120,6 +124,14 @@ impl EnrichmentConfig {
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
.unwrap_or_else(|| "en".to_string()), .unwrap_or_else(|| "en".to_string()),
signer: match crate::claims::load_or_create_key() {
Ok(s) => Some(s),
Err(e) => {
warn!(target: "meta-feeder::enrich", error = %e,
"no signing key; meta-core writes will fail");
None
}
},
} }
} }
} }
@@ -218,7 +230,14 @@ impl Enricher {
&self.cfg.gateway_peer, &self.cfg.gateway_peer,
); );
let body = meta_core::build_metadata_body(record, &prov); let body = meta_core::build_metadata_body(record, &prov);
meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await meta_core::put_record(&self.http, &self.cfg.meta_core_url, self.signer()?, cid, &body).await
}
fn signer(&self) -> anyhow::Result<&crate::claims::Signer> {
self.cfg
.signer
.as_ref()
.ok_or_else(|| anyhow::anyhow!("no signing key (METAMESH_SIGNING_KEY / _FILE)"))
} }
/// Run the video enrichment pipeline for one resolved CID: filename-parser /// Run the video enrichment pipeline for one resolved CID: filename-parser
@@ -240,14 +259,14 @@ impl Enricher {
Ok(()) => { Ok(()) => {
// Poll for the merged fields so tmdb gets the clean title. // Poll for the merged fields so tmdb gets the clean title.
if let Some(merged) = self if let Some(merged) = self
.poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) .poll_until_field(cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
.await .await
{ {
meta = merged; meta = merged;
} else { } else {
debug!( debug!(
target: "meta-feeder::enrich", target: "meta-feeder::enrich",
cid, "filename-parser produced no originalTitle within timeout; \ cid, "filename-parser produced no parsedTitle within timeout; \
handing tmdb the base fields" handing tmdb the base fields"
); );
} }
@@ -324,7 +343,7 @@ impl Enricher {
// Poll the primary for filename-parser's clean title, then tmdb it. // Poll the primary for filename-parser's clean title, then tmdb it.
let merged = self let merged = self
.poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) .poll_until_field(&p.cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
.await .await
.unwrap_or_else(|| p.base.clone()); .unwrap_or_else(|| p.base.clone());
@@ -355,9 +374,14 @@ impl Enricher {
} }
// PUT is merge/upsert — this layers the show-level anchor onto each // PUT is merge/upsert — this layers the show-level anchor onto each
// file's record without disturbing its per-file fields. // file's record without disturbing its per-file fields.
if let Err(e) = let res = match self.signer() {
meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await Ok(signer) => {
{ meta_core::put_record(&self.http, &self.cfg.meta_core_url, signer, &v.cid, &anchor)
.await
}
Err(e) => Err(e),
};
if let Err(e) = res {
warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
} }
} }
+208
View File
@@ -728,6 +728,159 @@ pub fn extract_codec(title: &str) -> Option<String> {
Some(norm.to_string()) Some(norm.to_string())
} }
// ---------------------------------------------------------------------------
// Release facts — `sourceMedium`, `releaseGroup`, `editions/*`, `flags/*`
// (METADATA_KEYS §3). Durable properties of a release (unlike its seeders), so
// they belong on the record; title-derived and best-effort like `quality` and
// `codec`, so absence means "couldn't tell". Each returns a closed vocabulary
// and drops what it does not recognise. The same functions normalise a
// structured value an upstream already parsed (AIOStreams' `quality`
// "BluRay REMUX", its editions "Director's Cut"), so the two paths agree.
// ---------------------------------------------------------------------------
/// Separators to spaces, the way every extractor here reads a title. Hyphens
/// are kept: `WEB-DL`, `HD-Rip` and the `-GROUP` suffix need them.
fn spaced(title: &str) -> String {
title.replace(['.', '_'], " ")
}
/// The medium the release was taken from, one of `cam`, `ts`, `tc`, `scr`,
/// `bluray-remux`, `bluray`, `webrip`, `web-dl`, `hdrip`, `hdtv`, `dvd`.
///
/// Pre-release sources are tested first: `HDCAM` beats any `HD` token, and a
/// `DVDSCR` is a screener, not a DVD. The bare `CAM`, `TS` and `TC` tokens only
/// count in capitals: as words they are a film title (*Cam*, 2018), a file
/// extension and an initialism too often.
pub fn extract_source_medium(title: &str) -> Option<&'static str> {
static CI: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\b(?:hd-?cam|cam-?rip)\b", "cam"),
(r"(?i)\b(?:telesync|hd-?ts|pdvd)\b", "ts"),
(r"(?i)\b(?:telecine|hd-?tc)\b", "tc"),
(r"(?i)\b(?:dvd-?scr|screener|scr)\b", "scr"),
(r"(?i)\b(?:bd-?remux|remux)\b", "bluray-remux"),
(r"(?i)\b(?:blu-?ray|bd-?rip|br-?rip|bdmv|bd25|bd50)\b", "bluray"),
(r"(?i)\bweb-?rip\b", "webrip"),
(r"(?i)\b(?:web-?dl|web)\b", "web-dl"),
(r"(?i)\bhd-?rip\b", "hdrip"),
(r"(?i)\b(?:hdtv|hdtvrip|uhdtv|pdtv|sdtv|tvrip|dsr)\b", "hdtv"),
(r"(?i)\b(?:dvd-?rip|dvd-?r|dvd5|dvd9|dvd)\b", "dvd"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("source re compiles"), v))
.collect()
});
static CAPS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[(r"\bCAM\b", "cam"), (r"\bTS\b", "ts"), (r"\bTC\b", "tc")]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("caps re compiles"), v))
.collect()
});
static DVD_REMUX: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"(?i)\bdvd\S*\b.*\bremux\b|\bremux\b.*\bdvd").expect("dvd remux re"));
let t = spaced(title);
// A DVD remux is a DVD: `remux` alone names no disc.
if DVD_REMUX.is_match(&t) {
return Some("dvd");
}
CAPS.iter()
.find(|(re, _)| re.is_match(&t))
.or_else(|| CI.iter().find(|(re, _)| re.is_match(&t)))
.map(|(_, v)| *v)
}
/// The group that made the release: a leading `[Group]` (fansub style) or a
/// trailing `-GROUP` (scene style), the same two rules as meta-watch's
/// `parseVersion`. Trailing tracker tags (`[rarbg]`, `[TGx]`), a video
/// extension and a usenet `-xpost` are removed first; a candidate that is
/// really a format token (`WEB-DL` → `DL`, `x264`, `1080p`) is refused.
pub fn extract_release_group(title: &str) -> Option<String> {
static LEADING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"^\s*\[([^\[\]]{2,32})\]").expect("leading group re"));
static TRAILING: LazyLock<Regex> =
LazyLock::new(|| Regex::new(r"-\s*([A-Za-z0-9][A-Za-z0-9&@]{1,24})\s*$").expect("trailing group re"));
static TAIL_NOISE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"(?i)(?:\s*(?:\[[^\]]*\]|\([^)]*\))|\.(?:mkv|mp4|avi|m4v|ts|m2ts|webm|wmv|mov)|-xpost)+\s*$")
.expect("tail noise re")
});
static FORMAT: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(?:[xh]26[45]|hevc|avc|av1|\d{3,4}[pi]|s\d+(?:e\d+)?)$").expect("format re")
});
let not_a_format = |g: &str| {
let l = g.to_ascii_lowercase();
!(matches!(
l.as_str(),
"dl" | "rip" | "web" | "hd" | "hdr" | "dts" | "ac3" | "aac" | "remux" | "sample" | "eng" | "multi"
) || l.chars().all(|c| c.is_ascii_digit())
|| FORMAT.is_match(&l))
};
if let Some(g) = LEADING.captures(title).map(|c| c[1].trim().to_string()) {
// `[1080p]` or `[HEVC]` at the front is a tag, not a group.
if not_a_format(&g) && extract_quality(&g).is_none() {
return Some(g);
}
}
let mut t = title.trim().to_string();
loop {
let cut = TAIL_NOISE.replace(&t, "").trim_end().to_string();
if cut == t {
break;
}
t = cut;
}
TRAILING
.captures(&t)
.map(|c| c[1].to_string())
.filter(|g| not_a_format(g))
}
/// The cut of the work this release carries — `editions/<x>` members. A
/// different cut has a different runtime, so these matter beyond a label
/// (a subtitle synced to the theatrical cut drifts on the extended one).
pub fn extract_editions(title: &str) -> BTreeSet<&'static str> {
static EDITIONS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\bextended\b", "extended"),
(r"(?i)\bdirector'?s?[\s-]*cut\b", "directors-cut"),
(r"(?i)\btheatrical\b", "theatrical"),
(r"(?i)\bunrated\b", "unrated"),
(r"(?i)\buncut\b", "uncut"),
(r"(?i)\buncensored\b", "uncensored"),
(r"(?i)\bimax\b", "imax"),
(r"(?i)\bremaster(?:ed)?\b", "remastered"),
(r"(?i)\bcriterion\b", "criterion"),
(r"(?i)\banniversary\b", "anniversary"),
(r"(?i)\bcollector'?s\b", "collectors"),
(r"(?i)\bultimate[\s-]+(?:edition|cut)\b", "ultimate"),
(r"(?i)\bspecial[\s-]+edition\b", "special"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("edition re compiles"), v))
.collect()
});
let t = spaced(title);
EDITIONS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
}
/// Properties of the encode itself — `flags/<x>` members: `repack` and
/// `proper` (a fixed re-release that supersedes the first), `upscaled` (the
/// resolution is not the source's: a 2160 upscale is not 4K), `regraded`.
pub fn extract_flags(title: &str) -> BTreeSet<&'static str> {
static FLAGS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"(?i)\b(?:repack\d?|rerip)\b", "repack"),
(r"(?i)\bproper\b", "proper"),
(r"(?i)\b(?:ai[\s-]*)?upscaled?\b", "upscaled"),
(r"(?i)\bregraded?\b", "regraded"),
]
.into_iter()
.map(|(re, v)| (Regex::new(re).expect("flag re compiles"), v))
.collect()
});
let t = spaced(title);
FLAGS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
}
/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + /// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
/// `cleanSpace`, in that order. /// `cleanSpace`, in that order.
fn preclean(filename: &str) -> String { fn preclean(filename: &str) -> String {
@@ -935,6 +1088,61 @@ pub fn clean_title(raw: &str) -> String {
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn source_medium_reads_real_release_names() {
let m = extract_source_medium;
assert_eq!(m("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR"), Some("bluray-remux"));
assert_eq!(m("The.Matrix.1999.1080p.BrRip.x264.YIFY.mp4"), Some("bluray"));
assert_eq!(m("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv"), Some("web-dl"));
assert_eq!(m("Show.S01E01.1080p.WEB.H264-GROUP"), Some("web-dl"));
assert_eq!(m("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), Some("webrip"));
assert_eq!(m("Movie 2023 HC HD-Rip 720p"), Some("hdrip"));
assert_eq!(m("Some Show [2x10] HDTV"), Some("hdtv"));
assert_eq!(m("Old.Film.1985.DVDRip.XviD"), Some("dvd"));
assert_eq!(m("Old.Film.1985.DVD.REMUX"), Some("dvd"));
assert_eq!(m("New.Film.2024.HDCAM.x264"), Some("cam"));
assert_eq!(m("New Film 2024 CAM"), Some("cam"));
assert_eq!(m("New.Film.2024.DVDSCR.x264"), Some("scr"));
assert_eq!(m("New.Film.2024.TELESYNC"), Some("ts"));
// AIOStreams' structured vocabulary goes through the same door.
assert_eq!(m("BluRay REMUX"), Some("bluray-remux"));
assert_eq!(m("WEB-DL"), Some("web-dl"));
assert_eq!(m("TC"), Some("tc"));
// The film "Cam" and a lower-case "ts" are not sources.
assert_eq!(m("Cam (2018) 1080p"), None);
assert_eq!(m("its a ts file"), None);
assert_eq!(m("Movie 2019 1080p x264"), None);
}
#[test]
fn release_group_is_the_leading_bracket_or_trailing_dash() {
let g = |t: &str| extract_release_group(t);
assert_eq!(g("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)").as_deref(), Some("yolerejiju"));
assert_eq!(g("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv").as_deref(), Some("G66"));
assert_eq!(g("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR").as_deref(), Some("FraMeSToR"));
assert_eq!(g("Oppenheimer.2023.1080p.BluRay.DD5.1.x264-GalaxyRG.mkv").as_deref(), Some("GalaxyRG"));
assert_eq!(g("Movie.2020.1080p.WEB-DL.x264-NTb[rarbg]").as_deref(), Some("NTb"));
assert_eq!(g("Breaking.Bad.S02E04.Down.1080p.BluRay.x264-OFT.mkv-xpost").as_deref(), Some("OFT"));
assert_eq!(g("Movie 2020 1080p WEB-DL [TGx]").as_deref(), None, "WEB-DL is not a group called DL");
assert_eq!(g("[1080p] Movie - 2020").as_deref(), None);
assert_eq!(g("Movie.2020.1080p.x264").as_deref(), None);
}
#[test]
fn editions_and_flags() {
let e = |t: &str| extract_editions(t).into_iter().collect::<Vec<_>>();
assert_eq!(e("Aliens.1986.Directors.Cut.1080p.BluRay"), ["directors-cut"]);
assert_eq!(e("LOTR.Fellowship.2001.EXTENDED.REMASTERED.2160p"), ["extended", "remastered"]);
assert_eq!(e("Director's Cut"), ["directors-cut"]);
assert_eq!(e("Ultimate Edition"), ["ultimate"]);
assert_eq!(e("Movie.2019.1080p.WEB-DL"), Vec::<&str>::new());
let f = |t: &str| extract_flags(t).into_iter().collect::<Vec<_>>();
assert_eq!(f("Game of Thrones S01E03 Lord Snow REPACK 1080p"), ["repack"]);
assert_eq!(f("Show.S01E01.PROPER.720p.HDTV"), ["proper"]);
assert_eq!(f("Movie 1999 2160p AI Upscaled Regrade"), ["regraded", "upscaled"]);
assert_eq!(f("The Proper Way 2020"), ["proper"], "a title word is indistinguishable — accepted noise");
}
fn se(title: &str) -> (Option<String>, Option<String>) { fn se(title: &str) -> (Option<String>, Option<String>) {
let r = extract_season_episode(title); let r = extract_season_episode(title);
(r.season, r.episode) (r.season, r.episode)
+40
View File
@@ -12,6 +12,10 @@
//! With the `transport` feature it also carries the **transport-plugin** //! With the `transport` feature it also carries the **transport-plugin**
//! contract ([`transport`]): the same split, applied to meta-share — a //! contract ([`transport`]): the same split, applied to meta-share — a
//! protocol-agnostic hull and one sidecar per byte-moving protocol. //! protocol-agnostic hull and one sidecar per byte-moving protocol.
//!
//! [`beacon`] is beacon v2 local discovery. Both harnesses advertise through it;
//! with `default-features = false, features = ["beacon"]` it is all a meta-*
//! service pulls in.
// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it // `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it
// directly rather than boxing on every embedded-DB call. Boxing each would be // directly rather than boxing on every embedded-DB call. Boxing each would be
@@ -25,38 +29,74 @@
/// `metacore-app` service the AppStore feeder apps ship on the shared network. /// `metacore-app` service the AppStore feeder apps ship on the shared network.
pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000"; pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000";
#[cfg(feature = "beacon")]
pub mod beacon;
#[cfg(feature = "feeder")]
pub mod budget; pub mod budget;
#[cfg(any(feature = "feeder", feature = "claims"))]
pub mod claims;
#[cfg(feature = "feeder")]
pub mod cache; pub mod cache;
#[cfg(feature = "feeder")]
pub mod common; pub mod common;
#[cfg(feature = "feeder")]
pub mod config; pub mod config;
#[cfg(feature = "feeder")]
pub mod domain; pub mod domain;
#[cfg(feature = "feeder")]
pub mod enrich; pub mod enrich;
#[cfg(feature = "filename")]
pub mod filename_meta; pub mod filename_meta;
#[cfg(feature = "feeder")]
pub mod hash; pub mod hash;
#[cfg(feature = "feeder")]
pub mod lang; pub mod lang;
#[cfg(feature = "feeder")]
pub mod licence; pub mod licence;
/// Search-wire-transient `live/*` fields — always available, no dependencies.
pub mod live;
#[cfg(feature = "feeder")]
pub mod meta_core; pub mod meta_core;
#[cfg(feature = "feeder")]
pub mod plugin; pub mod plugin;
#[cfg(feature = "feeder")]
pub mod query; pub mod query;
#[cfg(feature = "feeder")]
pub mod query_eval; pub mod query_eval;
#[cfg(feature = "feeder")]
pub mod serve; pub mod serve;
#[cfg(feature = "transport")] #[cfg(feature = "transport")]
pub mod transport; pub mod transport;
#[cfg(feature = "feeder")]
pub mod types; pub mod types;
#[cfg(feature = "feeder")]
pub use budget::RateBudget; pub use budget::RateBudget;
#[cfg(feature = "feeder")]
pub use config::{ConfigField, ConfigSchema, FieldKind}; pub use config::{ConfigField, ConfigSchema, FieldKind};
#[cfg(feature = "feeder")]
pub use licence::licence_from_url; pub use licence::licence_from_url;
#[cfg(feature = "feeder")]
pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher}; pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher};
#[cfg(feature = "feeder")]
pub use meta_core::FeederStore; pub use meta_core::FeederStore;
#[cfg(feature = "feeder")]
pub use plugin::{ pub use plugin::{
upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry,
RedeemClaim, RedeemClaim,
}; };
#[cfg(feature = "feeder")]
pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter};
#[cfg(feature = "feeder")]
pub use serve::{ pub use serve::{
configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto,
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
RedeemsResponse, RedeemsResponse,
}; };
#[cfg(any(feature = "feeder", feature = "claims"))]
pub use claims::{ClaimValue, SignedClaim};
#[cfg(feature = "claims")]
pub use claims::{load_or_create_key, ClaimError, Signer};
#[cfg(feature = "feeder")]
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
+79
View File
@@ -0,0 +1,79 @@
//! Search-wire-transient fields: facts a feeder sees at query time that are
//! true only *now* — a torrent's seeders, its leechers — and so must never
//! reach the replicated `/file/{cid}` hash (METADATA_KEYS rule #4).
//!
//! They travel under one reserved prefix, [`LIVE_PREFIX`], so every layer that
//! persists a record strips the whole family without knowing each key:
//!
//! ```text
//! live/seeders = "42" int-string
//! live/leechers = "3" int-string
//! live/at = "1791726110" unix seconds the feeder observed them
//! ```
//!
//! A record without them means "unknown", never "zero": a replayed or
//! persisted record has none, and that is the honest answer.
//!
//! The feeder SDK strips them in [`crate::meta_core::build_metadata_body`]
//! (every feeder self-publish); meta-gateway and meta-search strip them on
//! their own write paths (see METADATA_KEYS "Search-wire-transient fields").
use std::collections::BTreeMap;
/// The reserved prefix of the transient family.
pub const LIVE_PREFIX: &str = "live/";
/// Swarm seeders, `int-string`.
pub const LIVE_SEEDERS: &str = "live/seeders";
/// Swarm leechers, `int-string`.
pub const LIVE_LEECHERS: &str = "live/leechers";
/// When the feeder observed the `live/*` values, unix seconds.
pub const LIVE_AT: &str = "live/at";
/// The bare swarm counters some feeders wrote before the `live/*` family
/// existed (tribler). Stripped with it: rule #4 bars them in any spelling.
pub const LEGACY_SWARM_KEYS: [&str; 3] = ["seeders", "leechers", "peers"];
/// Whether a field is search-wire-transient and must not be persisted.
pub fn is_transient(key: &str) -> bool {
key.starts_with(LIVE_PREFIX) || LEGACY_SWARM_KEYS.contains(&key)
}
/// Remove every transient field. Call on the copy about to be persisted, never
/// on the record still bound for the search wire.
pub fn strip_transient(fields: &mut BTreeMap<String, String>) {
fields.retain(|k, _| !is_transient(k));
}
/// The `live/at` value for a feeder observing counters right now.
pub fn now_secs() -> String {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn strips_the_family_and_the_legacy_counters_only() {
let mut f: BTreeMap<String, String> = [
("live/seeders", "42"),
("live/at", "1"),
("seeders", "7"),
("peers", "9"),
("fileName", "x.mkv"),
("livestream", "true"),
("source/live", "true"),
]
.into_iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
strip_transient(&mut f);
let keys: Vec<&str> = f.keys().map(String::as_str).collect();
assert_eq!(keys, ["fileName", "livestream", "source/live"]);
}
}
+119 -21
View File
@@ -12,10 +12,12 @@
//! written by the feeder is indistinguishable from one the dispatcher wrote — //! written by the feeder is indistinguishable from one the dispatcher wrote —
//! the meta-share ingester, meta-watch, and ranking all behave the same. //! the meta-share ingester, meta-watch, and ranking all behave the same.
//! //!
//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core //! Since meta-core v2.0.0 every write is a batch of **signed claims** to
//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the //! `POST /api/claims` (spec `docs/implementation/metacore-claims-model.md`):
//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}` //! each field becomes one MMUR2 vote signed by this feeder's own key. Votes from
//! merges accumulate on the same `/file/{cid}` hash. //! different writers never overwrite each other — meta-core merges them — so the
//! feeder's base write and the enrichment plugins' later claims accumulate on
//! the same cid.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@@ -23,6 +25,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use serde::Deserialize; use serde::Deserialize;
use tracing::warn; use tracing::warn;
use crate::claims::{now_ms, ClaimValue, SignedClaim, Signer};
use crate::plugin::{HashKind, HashOutcome}; use crate::plugin::{HashKind, HashOutcome};
use crate::types::{DiscoveryRecord, Hash}; use crate::types::{DiscoveryRecord, Hash};
@@ -80,6 +83,11 @@ pub fn build_metadata_body(
) -> BTreeMap<String, String> { ) -> BTreeMap<String, String> {
let mut body: BTreeMap<String, String> = BTreeMap::new(); let mut body: BTreeMap<String, String> = BTreeMap::new();
for (k, v) in &record.fields { for (k, v) in &record.fields {
// `live/*` (and the legacy bare swarm counters) are true only at query
// time; a self-publishing feeder must not freeze them into the hash.
if crate::live::is_transient(k) {
continue;
}
if k.starts_with("cid_") && !v.is_empty() { if k.starts_with("cid_") && !v.is_empty() {
body.insert(format!("cids/{v}"), "true".to_string()); body.insert(format!("cids/{v}"), "true".to_string());
} else { } else {
@@ -90,22 +98,76 @@ pub fn build_metadata_body(
body body
} }
/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into /// One per-claim verdict from `POST /api/claims`.
/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`). #[derive(Debug, Deserialize)]
pub async fn put_record( pub struct ClaimStatus {
pub status: String,
#[serde(default)]
pub reason: Option<String>,
}
/// `POST /api/claims` — submit already-signed claims (e.g. relayed verbatim).
/// Fails on a non-2xx answer or when any claim is refused as `bad_signature` /
/// `bad_request`; `stale` and `unchanged` are normal outcomes.
pub async fn post_claims(
http: &reqwest::Client, http: &reqwest::Client,
base_url: &str, base_url: &str,
cid: &str, claims: &[SignedClaim],
body: &BTreeMap<String, String>, ) -> anyhow::Result<Vec<ClaimStatus>> {
) -> anyhow::Result<()> { if claims.is_empty() {
let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid); return Ok(Vec::new());
let resp = http.put(&url).json(body).send().await?; }
let url = format!("{}/api/claims", base_url.trim_end_matches('/'));
let resp = http.post(&url).json(claims).send().await?;
let status = resp.status(); let status = resp.status();
if !status.is_success() { if !status.is_success() {
let text = resp.text().await.unwrap_or_default(); let text = resp.text().await.unwrap_or_default();
anyhow::bail!("meta-core PUT {url} returned {status}: {text}"); anyhow::bail!("meta-core POST {url} returned {status}: {text}");
} }
Ok(()) let verdicts: Vec<ClaimStatus> = resp.json().await?;
if let Some((i, bad)) = verdicts
.iter()
.enumerate()
.find(|(_, v)| v.status == "bad_signature" || v.status == "bad_request")
{
anyhow::bail!(
"meta-core refused claim {} ({}): {} {}",
i,
claims.get(i).map(|c| c.key.as_str()).unwrap_or("?"),
bad.status,
bad.reason.as_deref().unwrap_or("")
);
}
Ok(verdicts)
}
/// Sign every field of `body` as a text claim on `cid` (all at "now") and
/// submit them. The claims-model replacement for the v1 `PUT /api/metadata`.
pub async fn put_record(
http: &reqwest::Client,
base_url: &str,
signer: &Signer,
cid: &str,
body: &BTreeMap<String, String>,
) -> anyhow::Result<()> {
let claims = signer.sign_fields(cid, body, now_ms());
post_claims(http, base_url, &claims).await.map(|_| ())
}
/// Sign retractions of `keys` on `cid` (this signer's own votes only).
pub async fn retract_keys(
http: &reqwest::Client,
base_url: &str,
signer: &Signer,
cid: &str,
keys: &[String],
) -> anyhow::Result<()> {
let ts = now_ms();
let claims: Vec<SignedClaim> = keys
.iter()
.map(|k| signer.sign(cid, k, ClaimValue::Null, ts))
.collect();
post_claims(http, base_url, &claims).await.map(|_| ())
} }
/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`). /// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`).
@@ -157,27 +219,49 @@ pub struct FeederStore {
http: reqwest::Client, http: reqwest::Client,
meta_core_url: String, meta_core_url: String,
gateway_peer: String, gateway_peer: String,
signer: Signer,
} }
impl FeederStore { impl FeederStore {
pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self { pub fn new(
http: reqwest::Client,
meta_core_url: String,
gateway_peer: String,
signer: Signer,
) -> Self {
FeederStore { FeederStore {
http, http,
meta_core_url: meta_core_url.trim_end_matches('/').to_string(), meta_core_url: meta_core_url.trim_end_matches('/').to_string(),
gateway_peer, gateway_peer,
signer,
} }
} }
/// Build from the feeder env. `None` when `META_CORE_URL` is unset (the /// Build from the feeder env. `None` when `META_CORE_URL` is unset, or when
/// feeder then falls back to the legacy "return outcomes, gateway stores" /// no signing key is configured (`METAMESH_SIGNING_KEY` /
/// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`. /// `METAMESH_SIGNING_KEY_FILE`) — the feeder then falls back to the legacy
/// "return outcomes, gateway stores" path. `gateway_peer` ←
/// `META_GATEWAY_PEER_ID` / `HOSTNAME`.
pub fn from_env() -> Option<Self> { pub fn from_env() -> Option<Self> {
let meta_core_url = env_nonempty("META_CORE_URL")?; let meta_core_url = env_nonempty("META_CORE_URL")?;
let signer = match crate::claims::load_or_create_key() {
Ok(s) => s,
Err(e) => {
warn!(target: "meta-feeder::store", error = %e,
"META_CORE_URL set but no signing key; not self-publishing");
return None;
}
};
let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID") let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID")
.or_else(|| env_nonempty("HOSTNAME")) .or_else(|| env_nonempty("HOSTNAME"))
.unwrap_or_else(|| "gateway-feeder".to_string()); .unwrap_or_else(|| "gateway-feeder".to_string());
let http = crate::common::build_http_client(30, "meta-feeder-store", None); let http = crate::common::build_http_client(30, "meta-feeder-store", None);
Some(Self::new(http, meta_core_url, gateway_peer)) Some(Self::new(http, meta_core_url, gateway_peer, signer))
}
/// The uid this store signs as.
pub fn signer(&self) -> &Signer {
&self.signer
} }
pub fn meta_core_url(&self) -> &str { pub fn meta_core_url(&self) -> &str {
@@ -257,7 +341,8 @@ impl FeederStore {
} }
} }
/// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance. /// Metadata-only self-publish: the record's fields (+ provenance) as signed
/// claims to `POST /api/claims`.
async fn store_metadata_only( async fn store_metadata_only(
&self, &self,
upstream_id: &str, upstream_id: &str,
@@ -267,7 +352,7 @@ impl FeederStore {
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer); let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer);
let body = build_metadata_body(record, &prov); let body = build_metadata_body(record, &prov);
put_record(&self.http, &self.meta_core_url, cid, &body).await put_record(&self.http, &self.meta_core_url, &self.signer, cid, &body).await
} }
} }
@@ -300,6 +385,7 @@ mod tests {
("title".into(), "Big Buck Bunny".into()), ("title".into(), "Big Buck Bunny".into()),
("cid_btih_v1_file".into(), "bafkPERFILE".into()), ("cid_btih_v1_file".into(), "bafkPERFILE".into()),
]), ]),
claims: Vec::new(),
} }
} }
@@ -332,4 +418,16 @@ mod tests {
); );
assert!(body.contains_key("provenance")); assert!(body.contains_key("provenance"));
} }
/// A self-publish never freezes swarm counters, in either spelling.
#[test]
fn body_drops_transient_fields() {
let mut r = rec();
for (k, v) in [("live/seeders", "42"), ("live/at", "1"), ("seeders", "7"), ("leechers", "2")] {
r.fields.insert(k.into(), v.into());
}
let body = build_metadata_body(&r, &Provenance::now("tribler", "abc", "peer"));
assert!(body.keys().all(|k| !crate::live::is_transient(k)), "{body:?}");
assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny"));
}
} }
+4
View File
@@ -100,6 +100,10 @@ pub enum GatewaySearchEvent {
record_id: String, record_id: String,
set: BTreeMap<String, String>, set: BTreeMap<String, String>,
remove: Vec<String>, remove: Vec<String>,
/// Signed form of the patch: `set` as text claims, `remove` as
/// retractions. Empty from a feeder (the gateway core signs).
#[serde(default)]
claims: Vec<crate::claims::SignedClaim>,
}, },
/// Retract a previously-emitted `Base` record entirely. /// Retract a previously-emitted `Base` record entirely.
Drop { record_id: String }, Drop { record_id: String },
+45 -6
View File
@@ -278,8 +278,15 @@ fn gateway_error_response(e: GatewayError) -> Response {
// -------- handlers --------------------------------------------------------- // -------- handlers ---------------------------------------------------------
async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> { async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
let mut plugins: Vec<PluginManifest> = state Json(build_manifest(&state.plugins, &state.version))
.plugins }
/// The `/manifest` body — also what the beacon advertise's `rev` is a hash of.
fn build_manifest(
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
version: &str,
) -> ManifestResponse {
let mut plugins: Vec<PluginManifest> = plugins
.values() .values()
.map(|p| PluginManifest { .map(|p| PluginManifest {
id: p.upstream_id().to_string(), id: p.upstream_id().to_string(),
@@ -294,10 +301,27 @@ async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
}) })
.collect(); .collect();
plugins.sort_by(|a, b| a.id.cmp(&b.id)); plugins.sort_by(|a, b| a.id.cmp(&b.id));
Json(ManifestResponse { ManifestResponse {
feeder_version: state.version.clone(), feeder_version: version.to_string(),
plugins, plugins,
}) }
}
/// The feeder's beacon v2 resource: one `metamesh.feeder/<upstream_id>` cap per
/// hosted plugin, the manifest endpoint, and `rev` = hash of the manifest.
fn beacon_resource(
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
version: &str,
listen_port: u16,
) -> crate::beacon::Resource {
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
let m = build_manifest(plugins, version);
let mut r = Resource::new("feeder", m.plugins.iter().map(|p| caps::feeder(&p.id)))
.with_endpoint("http", advertise_url(listen_port))
.with_endpoint("manifest", "/manifest");
r.rev = Some(rev_of(&m));
r.binds = binds_from_env();
r
} }
async fn redeems(State(state): State<AppState>) -> Json<RedeemsResponse> { async fn redeems(State(state): State<AppState>) -> Json<RedeemsResponse> {
@@ -568,7 +592,8 @@ pub async fn serve_feeders(
let state_dir = state_dir.into(); let state_dir = state_dir.into();
let configured = configure_plugins(plugins, &state_dir)?; let configured = configure_plugins(plugins, &state_dir)?;
let version = env!("CARGO_PKG_VERSION").to_string(); let version = env!("CARGO_PKG_VERSION").to_string();
let app = router(configured, version, state_dir.clone()); let advertised = configured.clone();
let app = router(configured, version.clone(), state_dir.clone());
let listener = tokio::net::TcpListener::bind(listen).await?; let listener = tokio::net::TcpListener::bind(listen).await?;
info!( info!(
@@ -576,6 +601,18 @@ pub async fn serve_feeders(
%listen, %listen,
"feeder listening" "feeder listening"
); );
// Beacon v2: advertise once the port is bound, so a consumer that hears us
// can connect. The node lives for the process.
let port = listen.port();
let name = std::env::var("SERVICE_NAME").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| {
let mut ids: Vec<&String> = advertised.keys().collect();
ids.sort();
ids.first().map_or_else(|| "meta-feeder".to_string(), |id| format!("meta-feeder-{id}"))
});
let beacon_version = version.clone();
let _beacon = crate::beacon::advertise_plugin(&name, &version, move || {
beacon_resource(&advertised, &beacon_version, port)
});
axum::serve(listener, app) axum::serve(listener, app)
.await .await
.map_err(anyhow::Error::from) .map_err(anyhow::Error::from)
@@ -762,6 +799,7 @@ mod tests {
upstream_id: "stub".into(), upstream_id: "stub".into(),
record_id: "1".into(), record_id: "1".into(),
fields: BTreeMap::new(), fields: BTreeMap::new(),
claims: Vec::new(),
}; };
let events = vec![ let events = vec![
GatewaySearchEvent::Base(rec), GatewaySearchEvent::Base(rec),
@@ -769,6 +807,7 @@ mod tests {
record_id: "1".into(), record_id: "1".into(),
set: BTreeMap::from([("title".to_string(), "X".to_string())]), set: BTreeMap::from([("title".to_string(), "X".to_string())]),
remove: vec![], remove: vec![],
claims: vec![],
}, },
GatewaySearchEvent::Done, GatewaySearchEvent::Done,
]; ];
+19
View File
@@ -156,6 +156,25 @@ impl RemoteTransport {
} }
} }
/// `PUT /manifests/:cid`: hand a manifest-consuming plugin (contract 2) the
/// manifest a pointer resolved to, after it answered `428` +
/// [`super::dto::HDR_NEEDS_MANIFEST`].
pub async fn push_manifest(&self, cid: &str, manifest: bytes::Bytes) -> Result<(), ApiError> {
let r = self
.http
.put(self.url(&format!("/manifests/{cid}")))
.body(manifest)
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
if r.status().is_success() {
Ok(())
} else {
Err(error_from(r).await)
}
}
// ---- typed helpers for extra routes ------------------------------------ // ---- typed helpers for extra routes ------------------------------------
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> { pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
+31 -2
View File
@@ -9,8 +9,37 @@ use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
/// Bumped on any breaking change to the common routes below. The hull refuses /// Bumped on any breaking change to the common routes below. The hull refuses
/// a plugin whose `/manifest` reports a different major contract. /// a plugin whose `/manifest` reports a contract outside
pub const CONTRACT_VERSION: u32 = 1; /// [`MIN_CONTRACT`]`..=`[`CONTRACT_VERSION`].
///
/// - **2**: a plugin that consumes a manifest (nzb) no longer pulls it from the
/// hull. It answers `/raw` (and its verify route) with `428` +
/// [`HDR_NEEDS_MANIFEST`] until the hull has pushed one with
/// `PUT /manifests/:cid`.
pub const CONTRACT_VERSION: u32 = 2;
/// The oldest contract the hull still drives (contract-1 nzb plugins pull their
/// manifest from `/internal/nzb/manifest/:cid`, kept for one release).
pub const MIN_CONTRACT: u32 = 1;
/// `428` marker: this plugin needs the cid's manifest pushed before it can
/// answer. The hull resolves the pointer, `PUT`s `/manifests/:cid`, and retries.
pub const HDR_NEEDS_MANIFEST: &str = "x-metamesh-needs-manifest";
/// Why a pointer could not be resolved, on a non-2xx `/ipfs-tier/resolve`
/// answer: [`RESOLVE_NO_GATEWAY`], [`RESOLVE_UNCLAIMED`], [`RESOLVE_QUOTA`],
/// [`RESOLVE_NOT_FOUND`] or [`RESOLVE_UPSTREAM`].
pub const HDR_RESOLVE_ERROR: &str = "x-metamesh-resolve-error";
/// No gateway is known at all yet (discovery is eventual): retry.
pub const RESOLVE_NO_GATEWAY: &str = "no-gateway";
/// Gateways are known, but none claims this pointer's key.
pub const RESOLVE_UNCLAIMED: &str = "unclaimed";
/// The provider's quota is spent: retry after `Retry-After`.
pub const RESOLVE_QUOTA: &str = "quota";
/// Every claiming gateway answered not found (or nothing is stored and the
/// pointer cannot be redeemed).
pub const RESOLVE_NOT_FOUND: &str = "not-found";
/// A gateway failed, or answered bytes that did not verify.
pub const RESOLVE_UPSTREAM: &str = "upstream";
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request. /// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
pub const HDR_LANE: &str = "x-metamesh-lane"; pub const HDR_LANE: &str = "x-metamesh-lane";
+4 -3
View File
@@ -39,10 +39,11 @@ use super::dto::Event;
/// Default hull callback base, matching the dev and store compose service name. /// Default hull callback base, matching the dev and store compose service name.
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001"; pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
/// Where the hull got a manifest: the record's `manifest` pointer, or a /// Where a pointer's bytes came from: `store` (already held here, named by the
/// gateway redeem. /// record's pointer field), `redeem` (a gateway), or `record` (contract-1
/// callback naming for `store`).
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source"; pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
/// The content cid of a `record`-sourced manifest. /// The content cid the pointer resolved to.
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid"; pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
/// `.nzb` bytes as the hull resolved them. /// `.nzb` bytes as the hull resolved them.
+10
View File
@@ -17,6 +17,7 @@
//! |---|---| //! |---|---|
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` | //! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) | //! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
//! | `GET resolve/:cid?hint=` → bytes | a [pointer](super::pointer)'s bytes: the stored copy `hint` names, else a gateway redeem (see below) |
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` | //! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` | //! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` | //! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
@@ -40,6 +41,15 @@
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for //! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
//! a display name. //! a display name.
//! **Resolving a pointer** (`GET resolve/:cid`) is the one route that reaches
//! outside the swarm: a pointer is answered only by a gateway holding the key,
//! over plain HTTP at the gateway's base URL. Gateways come from config (the
//! local one, pinned) and from the swarm directory; the fastest one whose redeem
//! claim covers the pointer is asked first. `2xx` carries the bytes plus
//! [`HDR_MANIFEST_SOURCE`](super::hull::HDR_MANIFEST_SOURCE) (`store` | `redeem`)
//! and [`HDR_MANIFEST_CID`](super::hull::HDR_MANIFEST_CID); a failure carries
//! [`HDR_RESOLVE_ERROR`](super::dto::HDR_RESOLVE_ERROR).
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)] #[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
+3 -1
View File
@@ -22,6 +22,7 @@
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers. //! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade). //! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings. //! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
//! - [`pointer`]: which cids are pointers (resolved before their bytes exist).
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`. //! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
//! - `testkit` (feature `testkit`): conformance checks for any implementation. //! - `testkit` (feature `testkit`): conformance checks for any implementation.
//! //!
@@ -42,6 +43,7 @@ pub mod ipfs;
pub mod magic; pub mod magic;
pub mod nzb; pub mod nzb;
pub mod plugin; pub mod plugin;
pub mod pointer;
pub mod range; pub mod range;
pub mod serve; pub mod serve;
pub mod torrent; pub mod torrent;
@@ -52,7 +54,7 @@ pub use client::RemoteTransport;
pub use config::ConfigPlane; pub use config::ConfigPlane;
pub use dto::{ pub use dto::{
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport, Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
ReconcileRequest, CONTRACT_VERSION, ReconcileRequest, CONTRACT_VERSION, MIN_CONTRACT,
}; };
pub use error::ApiError; pub use error::ApiError;
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane}; pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
+248
View File
@@ -0,0 +1,248 @@
//! **Pointer cids** — cids that name something to fetch rather than hash it.
//!
//! A pointer's multihash is `identity`: the cid *contains* the address (an
//! indexer host + release id, a provider source + file id), not a digest of the
//! bytes. Bitswap cannot answer one, because there is no hash to verify against.
//! It is resolved instead: by a manifest this peer already stored (the record's
//! pointer field names its content cid), or by a **redeem** at a gateway whose
//! feeder holds the credential for `key`.
//!
//! This module is the single place that knows which codecs are pointers, so the
//! hull ("resolve it first") and the ipfs plugin ("here is how") agree.
//!
//! | codec | what it points at | redeem key | record field |
//! |---|---|---|---|
//! | `0x1005` nzb-release | a `.nzb` at an indexer | indexer host | `manifest` |
//! | `0x1003` nzb-posting | a self-scanned `.nzb` | — (record only) | `manifest` |
//! | `0x100A` provider-file | a file a provider holds | provider source | `file` |
use anyhow::{Context, Result};
use super::cid::{parse_record_cid, read_uvarint};
use super::nzb::{
decode_nzb_release_cid, normalize_indexer_host, NZB_POSTING_CODEC, NZB_RELEASE_CODEC,
};
/// MetaMesh-private multicodec for a provider-held file (`docs/cid-formats.md` §8).
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
/// Redeem-claim codec names, as gateways advertise them in
/// `GET /api/gateway/plugins` (`redeems[].codec`).
pub const REDEEM_NZB_RELEASE: &str = "nzb-release";
pub const REDEEM_PROVIDER_FILE: &str = "provider-file";
/// Record fields a redeem merges onto the pointer's record: the content cid of
/// what it resolved to.
pub const FIELD_MANIFEST: &str = "manifest";
pub const FIELD_FILE: &str = "file";
/// Identity multihash code: the digest bytes are the payload verbatim.
const IDENTITY_MULTIHASH: u64 = 0x00;
/// A decoded pointer.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Pointer {
/// The redeem-claim codec, or `None` when no gateway can redeem it (a
/// self-scanned posting resolves from the record alone).
pub redeem: Option<RedeemKey>,
/// The record field naming the resolved content cid.
pub field: &'static str,
/// What the resolved bytes are: a manifest a byte plugin consumes, or the
/// file itself.
pub kind: PointerKind,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RedeemKey {
/// [`REDEEM_NZB_RELEASE`] or [`REDEEM_PROVIDER_FILE`].
pub codec: &'static str,
/// The indexer host / provider source a gateway's claim must cover.
pub key: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PointerKind {
/// A `.nzb`: handed to the nzb plugin, which fetches the bytes.
NzbManifest,
/// The file itself (a subtitle): served as is.
File,
}
/// Decode `cid` as a pointer.
///
/// - `None` — not a pointer: an ordinary content cid, or unparseable.
/// - `Some(Err(_))` — the codec says pointer but the payload is malformed; the
/// caller answers `400`.
pub fn decode(cid: &str) -> Option<Result<Pointer>> {
match codec_of(cid)? {
NZB_RELEASE_CODEC => Some(decode_nzb_release(cid)),
NZB_POSTING_CODEC => Some(Ok(Pointer {
redeem: None,
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})),
PROVIDER_FILE_CODEC => Some(decode_provider_file(cid).map(|(source, _id)| Pointer {
redeem: Some(RedeemKey { codec: REDEEM_PROVIDER_FILE, key: source }),
field: FIELD_FILE,
kind: PointerKind::File,
})),
_ => None,
}
}
/// Is `cid` a pointer (well-formed or not)?
pub fn is_pointer(cid: &str) -> bool {
decode(cid).is_some()
}
fn decode_nzb_release(cid: &str) -> Result<Pointer> {
let mscid = parse_record_cid(cid)?;
let locator = decode_nzb_release_cid(&mscid)?;
Ok(Pointer {
redeem: Some(RedeemKey {
codec: REDEEM_NZB_RELEASE,
key: normalize_indexer_host(locator.authority()),
}),
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})
}
/// Decode a `provider-file` cid into `(source, id)`. Errors on a different
/// codec or a malformed payload. Works on the raw multibase string because a
/// long provider id can exceed the 64-byte multihash a typed cid parse allows.
pub fn decode_provider_file(cid: &str) -> Result<(String, String)> {
let bytes = raw_cid_bytes(cid).context("provider-file: not a base32 CIDv1")?;
let mut at = 0usize;
let next = |at: &mut usize, what: &str| -> Result<u64> {
let (v, n) = read_uvarint(&bytes[*at..])
.with_context(|| format!("provider-file: {what} varint"))?;
*at += n;
Ok(v)
};
anyhow::ensure!(next(&mut at, "version")? == 1, "provider-file: not a CIDv1");
anyhow::ensure!(next(&mut at, "codec")? == PROVIDER_FILE_CODEC, "not a provider-file cid");
let mh = next(&mut at, "multihash-code")?;
anyhow::ensure!(
mh == IDENTITY_MULTIHASH,
"provider-file locator must use the identity multihash (0x00), got 0x{mh:x}"
);
let len = next(&mut at, "digest-length")?;
anyhow::ensure!(
(bytes.len() - at) as u64 == len,
"provider-file digest length {len} != {} trailing byte(s)",
bytes.len() - at
);
let source_len = next(&mut at, "source-length")?;
let rest = &bytes[at..];
let source_len = usize::try_from(source_len)
.ok()
.filter(|n| *n <= rest.len())
.context("provider-file: source length overruns the digest")?;
let (source, id) = rest.split_at(source_len);
let source = std::str::from_utf8(source).context("provider-file source is not utf-8")?;
let id = std::str::from_utf8(id).context("provider-file id is not utf-8")?;
anyhow::ensure!(
!source.is_empty() && !id.is_empty(),
"provider-file locator needs both a source and an id"
);
Ok((source.to_string(), id.to_string()))
}
/// The codec of a base32 CIDv1 string, without the 64-byte multihash limit.
fn codec_of(cid: &str) -> Option<u64> {
let bytes = raw_cid_bytes(cid)?;
let (version, n) = read_uvarint(&bytes).ok()?;
if version != 1 {
return None;
}
read_uvarint(&bytes[n..]).ok().map(|(codec, _)| codec)
}
/// Decode a `b`-prefixed lowercase base32 (RFC 4648, no padding) CID string.
fn raw_cid_bytes(cid: &str) -> Option<Vec<u8>> {
let body = cid.strip_prefix('b')?;
let mut out = Vec::with_capacity(body.len() * 5 / 8);
let (mut acc, mut bits) = (0u32, 0u32);
for c in body.bytes() {
let v = match c {
b'a'..=b'z' => c - b'a',
b'2'..=b'7' => c - b'2' + 26,
_ => return None,
} as u32;
acc = (acc << 5) | v;
bits += 5;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// The Ragna Crimson release from watch.nsl.sh (2026-10-04).
const NZBGEEK: &str = "bagcsaabfcrqxa2jonz5gez3fmvvs42lomzxs6ylqngxuch22nhfhl4v4vmyhiegah2ka";
fn b32(bytes: &[u8]) -> String {
const A: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
let (mut acc, mut bits, mut s) = (0u32, 0u32, String::from("b"));
for &b in bytes {
acc = (acc << 8) | b as u32;
bits += 8;
while bits >= 5 {
bits -= 5;
s.push(A[((acc >> bits) & 31) as usize] as char);
}
acc &= (1 << bits) - 1;
}
if bits > 0 {
s.push(A[((acc << (5 - bits)) & 31) as usize] as char);
}
s
}
fn provider_file(source: &str, id: &str) -> String {
let mut digest = vec![source.len() as u8];
digest.extend_from_slice(source.as_bytes());
digest.extend_from_slice(id.as_bytes());
// version 1, codec 0x100A (varint 0x8a 0x20), identity, length
let mut wire = vec![0x01, 0x8a, 0x20, 0x00, digest.len() as u8];
wire.extend_from_slice(&digest);
b32(&wire)
}
#[test]
fn an_nzb_release_is_a_pointer_keyed_by_its_indexer_host() {
let p = decode(NZBGEEK).unwrap().unwrap();
assert_eq!(
p.redeem,
Some(RedeemKey { codec: REDEEM_NZB_RELEASE, key: "api.nzbgeek.info".into() })
);
assert_eq!(p.field, FIELD_MANIFEST);
assert_eq!(p.kind, PointerKind::NzbManifest);
}
#[test]
fn a_provider_file_is_a_pointer_keyed_by_its_source() {
let cid = provider_file("opensubtitles", "file:7061834");
assert_eq!(
decode_provider_file(&cid).unwrap(),
("opensubtitles".into(), "file:7061834".into())
);
let p = decode(&cid).unwrap().unwrap();
assert_eq!(p.redeem.unwrap().key, "opensubtitles");
assert_eq!(p.field, FIELD_FILE);
assert_eq!(p.kind, PointerKind::File);
}
#[test]
fn a_content_cid_is_not_a_pointer() {
assert!(!is_pointer("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy"));
assert!(!is_pointer("not a cid"));
}
}
+32 -1
View File
@@ -57,16 +57,47 @@ pub async fn serve_transport<P: TransportPlugin>(
plugin: Arc<P>, plugin: Arc<P>,
listen: SocketAddr, listen: SocketAddr,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let id = plugin.manifest().id; let manifest = plugin.manifest();
let id = manifest.id.clone();
let advertised = Arc::clone(&plugin);
let app = router(plugin); let app = router(plugin);
let listener = tokio::net::TcpListener::bind(listen).await?; let listener = tokio::net::TcpListener::bind(listen).await?;
info!(%listen, plugin = %id, "transport plugin listening"); info!(%listen, plugin = %id, "transport plugin listening");
// Beacon v2: advertise once bound; say `bye` on clean shutdown so the hull
// drops us at once instead of after the liveness window.
let port = listen.port();
// `node.name` is a short machine name; the human description stays in the
// manifest (`implementation`), which a consumer fetches over HTTP.
let beacon_name = std::env::var("SERVICE_NAME")
.ok()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("meta-transport-{}", manifest.id));
let beacon = crate::beacon::advertise_plugin(&beacon_name, &manifest.version, move || {
beacon_resource(advertised.as_ref(), port)
});
axum::serve(listener, app) axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal()) .with_graceful_shutdown(shutdown_signal())
.await?; .await?;
if let Some(b) = beacon {
b.bye().await;
}
Ok(()) Ok(())
} }
/// The transport's beacon v2 resource: id = manifest id, cap
/// `metamesh.transport/<id>@<contract>`, `rev` = hash of the served manifest.
fn beacon_resource<P: TransportPlugin + ?Sized>(plugin: &P, listen_port: u16) -> crate::beacon::Resource {
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
let mut m = plugin.manifest();
m.config = plugin.config().is_some();
let mut r = Resource::new(m.id.clone(), [caps::transport(&m.id, m.contract)])
.with_endpoint("http", advertise_url(listen_port))
.with_endpoint("manifest", "/manifest");
r.rev = Some(rev_of(&m));
r.binds = binds_from_env();
r
}
/// Resolves on SIGTERM (docker stop) or Ctrl-C. /// Resolves on SIGTERM (docker stop) or Ctrl-C.
pub async fn shutdown_signal() { pub async fn shutdown_signal() {
let ctrl_c = async { let ctrl_c = async {
+8
View File
@@ -49,6 +49,13 @@ pub struct DiscoveryRecord {
pub upstream_id: String, pub upstream_id: String,
pub record_id: String, pub record_id: String,
pub fields: BTreeMap<String, String>, pub fields: BTreeMap<String, String>,
/// The record's MMUR2 claims (spec §10). Empty from a feeder that returns
/// plain `fields` (the gateway core signs those as itself); filled by a
/// self-publishing feeder or by the gateway before the record goes on the
/// wire. `#[serde(default)]` keeps older feeders' JSON `/query` answers
/// decoding.
#[serde(default)] // never skip_serializing_if: bincode is positional
pub claims: Vec<crate::claims::SignedClaim>,
} }
/// In-process error returned by `FeederPlugin` methods. /// In-process error returned by `FeederPlugin` methods.
@@ -102,6 +109,7 @@ mod tests {
("title".to_string(), "AlphaFold".to_string()), ("title".to_string(), "AlphaFold".to_string()),
("year".to_string(), "2021".to_string()), ("year".to_string(), "2021".to_string()),
]), ]),
claims: Vec::new(),
}; };
let j = serde_json::to_string(&r).expect("serialize"); let j = serde_json::to_string(&r).expect("serialize");
let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize"); let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize");
+93
View File
@@ -0,0 +1,93 @@
{
"_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
"privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"vectors": [
{
"name": "text",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "title",
"value": "The Matrix",
"ts": 1793001234567,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087",
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8"
},
{
"name": "empty-text",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "title",
"value": "",
"ts": 1,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001",
"sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642"
},
{
"name": "unicode",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "titles/jpn/攻殻機動隊",
"value": "true",
"ts": 1793001234567,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087",
"sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207"
},
{
"name": "keyset-member",
"cid": "bagacbabaec7v3",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "cids/bafkreih5kapbjzq",
"value": "true",
"ts": 23,
"signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517",
"sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761"
},
{
"name": "bytes",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "seek",
"value": null,
"valueBytesHex": "00ff10ab",
"ts": 24,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818",
"sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4"
},
{
"name": "retraction",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "genres/Drama",
"value": null,
"ts": 255,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff",
"sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac"
},
{
"name": "long-value",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "plot/eng",
"value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"ts": 65536,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000",
"sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2"
},
{
"name": "ts-64bit",
"cid": "bafkreih5kapbjzq",
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
"key": "year",
"value": "1999",
"ts": 1099511627776,
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000",
"sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d"
}
],
"highS": {
"note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it",
"vector": "text",
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899"
}
}