Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f804d3adb1 | ||
|
|
01cc4e15a5 | ||
|
|
38ee6d8e41 | ||
|
|
612f2aed66 | ||
|
|
027917c8cf | ||
|
|
26beb2444b |
+48
-15
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "meta-feeder-sdk"
|
||||
version = "1.5.0"
|
||||
version = "1.10.0"
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
@@ -32,44 +32,77 @@ publish = false
|
||||
# of here so feeders that don't need them don't pay for them.
|
||||
|
||||
[features]
|
||||
default = ["feeder"]
|
||||
# Everything a feeder sidecar (and the transport contract) builds on: the
|
||||
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
|
||||
# so existing consumers build exactly what they always did.
|
||||
feeder = [
|
||||
"beacon", "filename", "claims",
|
||||
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
|
||||
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
|
||||
"dep:redb",
|
||||
]
|
||||
# Release-name parsing (`src/filename_meta.rs`: clean_title, season/episode,
|
||||
# quality, codec). Needs only `regex`, so a client (meta-watch) can derive a
|
||||
# display name from a raw release string without the feeder machinery.
|
||||
filename = ["dep:regex"]
|
||||
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
|
||||
# so a meta-* service takes the SDK with `default-features = false,
|
||||
# features = ["beacon"]` and pulls in none of the feeder machinery.
|
||||
beacon = ["dep:libc"]
|
||||
# The meta-share transport-plugin contract (`src/transport/`): the hull's
|
||||
# client, the plugins' axum harness, the focus replica, shared error/range
|
||||
# types. Off by default so feeders build exactly what they always did.
|
||||
transport = ["dep:cid", "dep:libc", "dep:tokio-util"]
|
||||
transport = ["feeder", "dep:cid", "dep:tokio-util"]
|
||||
# Conformance checks any transport implementation must pass.
|
||||
testkit = ["transport"]
|
||||
# MMUR2 signed claims (`src/claims.rs`): the `SignedClaim` type, its canonical
|
||||
# bytes, signing (k256, compact low-S), verification and the writer key
|
||||
# (`METAMESH_SIGNING_KEY` / `_FILE`). `feeder` enables it because the gateway
|
||||
# wire (`DiscoveryRecord`, `EnrichPatch`) carries claims; a meta-* service that
|
||||
# only needs claims takes `default-features = false, features = ["claims"]`.
|
||||
claims = [
|
||||
"dep:k256", "dep:bs58", "dep:sha2", "dep:base64", "dep:rand_core",
|
||||
"dep:anyhow", "dep:thiserror",
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
# Async runtime + HTTP server (the serve_feeders harness)
|
||||
tokio = { version = "1.40", features = ["full"] }
|
||||
axum = { version = "0.7", features = ["json"] }
|
||||
hyper = { version = "1", features = ["full"] }
|
||||
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
|
||||
axum = { version = "0.7", features = ["json"], optional = true }
|
||||
hyper = { version = "1", features = ["full"], optional = true }
|
||||
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false, optional = true }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
base64 = "0.22"
|
||||
base64 = { version = "0.22", optional = true }
|
||||
|
||||
# Utilities
|
||||
futures = "0.3"
|
||||
async-trait = "0.1"
|
||||
anyhow = "1"
|
||||
thiserror = "1"
|
||||
futures = { version = "0.3", optional = true }
|
||||
async-trait = { version = "0.1", optional = true }
|
||||
anyhow = { version = "1", optional = true }
|
||||
thiserror = { version = "1", optional = true }
|
||||
tracing = "0.1"
|
||||
sha2 = "0.10"
|
||||
bytes = "1"
|
||||
sha2 = { version = "0.10", optional = true }
|
||||
bytes = { version = "1", optional = true }
|
||||
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
|
||||
regex = "1"
|
||||
regex = { version = "1", optional = true }
|
||||
|
||||
# MMUR2 claims (the `claims` feature).
|
||||
k256 = { version = "0.13", features = ["ecdsa", "std"], optional = true }
|
||||
bs58 = { version = "0.5", optional = true }
|
||||
rand_core = { version = "0.6", features = ["getrandom"], optional = true }
|
||||
|
||||
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
|
||||
# cache.rs. Pure-Rust single-process embedded store.
|
||||
redb = "2"
|
||||
redb = { version = "2", optional = true }
|
||||
|
||||
# CID parsing for the transport contract only (`transport::cid`). Optional so
|
||||
# feeders don't pull it.
|
||||
cid = { version = "0.11", optional = true }
|
||||
# statvfs(2) for `transport::fsutil::available_bytes`.
|
||||
# Raw sockets for `beacon` (per-interface multicast) and statvfs(2) for
|
||||
# `transport::fsutil::available_bytes`.
|
||||
libc = { version = "0.2", optional = true }
|
||||
# `ReaderStream` for `transport::file`.
|
||||
tokio-util = { version = "0.7", features = ["io"], optional = true }
|
||||
|
||||
@@ -35,6 +35,42 @@ async fn main() -> anyhow::Result<()> {
|
||||
A **service is a `Vec`** — grouping plugins into one binary is a deployment
|
||||
choice, not an architectural one.
|
||||
|
||||
### Discovery (1.6.0, scan 1.7.0)
|
||||
|
||||
`serve_feeders` and `serve_transport` advertise the process over **beacon v2**
|
||||
(UDP multicast `239.255.99.1:9099`, spec: meta-root
|
||||
`docs/project-architecture/beacon-v2.md`) as soon as the port is bound — a
|
||||
feeder as `metamesh.feeder/<upstream_id>` (one cap per hosted plugin), a
|
||||
transport as `metamesh.transport/<id>@<contract>` — with `endpoints.http`,
|
||||
`endpoints.manifest = /manifest` and `rev` = a hash of the served manifest
|
||||
(re-advertised when it changes). Nothing to call. Env:
|
||||
|
||||
| Env | Default | |
|
||||
|---|---|---|
|
||||
| `ENABLE_UDP_DISCOVERY` | `true` | `false` turns advertising off |
|
||||
| `BEACON_ADVERTISE_URL` | `http://$HOSTNAME:<port>` | `endpoints.http` |
|
||||
| `BEACON_BINDS` | unset | the one consumer instance allowed to list this plugin |
|
||||
| `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS` | `239.255.99.1` / `9099` / `10000` | |
|
||||
|
||||
`beacon::scan(node, pattern, configured, wait)` (1.7.0) is what a consumer's
|
||||
**Scan** button runs: probe everyone, wait, then classify every resource
|
||||
matching `pattern` as `configured` / `addable` / `bound-elsewhere` against the
|
||||
consumer's `(name, url)` list — the `ScanReport` JSON the shared
|
||||
`<meta-beacon-scan>` element renders. Transports advertise as
|
||||
`meta-transport-<id>` (`SERVICE_NAME` overrides), feeders as
|
||||
`meta-feeder-<first upstream>`.
|
||||
|
||||
The `beacon` module is also usable on its own — the meta-* services depend on
|
||||
the SDK for nothing else:
|
||||
|
||||
```toml
|
||||
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0", default-features = false, features = ["beacon"] }
|
||||
```
|
||||
|
||||
Features: `feeder` (default — the whole feeder harness; implies `beacon`),
|
||||
`beacon` (tokio/serde/tracing/libc only), `transport` (implies `feeder`),
|
||||
`testkit`.
|
||||
|
||||
## Writing a plugin
|
||||
|
||||
Four required methods; everything else has a default:
|
||||
@@ -141,6 +177,7 @@ and `provider-file` (`0x100A`, `(source, id)` — `docs/cid-formats.md` §8).
|
||||
| `hash` | the content-addressing families above |
|
||||
| `filename_meta` / `lang` | release-tag parsing helpers |
|
||||
| `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract |
|
||||
| `beacon` | beacon v2 local discovery: `BeaconNode`, `Resource`, `cap_matches`, the MetaMesh `caps` vocabulary |
|
||||
|
||||
## Development
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
//! Beacon v2 — local resource advertise / discover over UDP multicast
|
||||
//! (`239.255.99.1:9099`).
|
||||
//!
|
||||
//! Every meta-* service and every SDK-hosted plugin runs one [`BeaconNode`]: it
|
||||
//! advertises this process's [`Resource`]s, each tagged with capability strings
|
||||
//! (`metamesh.service/meta-sort`, `metamesh.transport/nzb@1`, …), and keeps a
|
||||
//! live view of everyone else's, which consumers filter by capability
|
||||
//! ([`cap_matches`]).
|
||||
//!
|
||||
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`
|
||||
//! (the Go / TypeScript / Python ports are written against it too). This module
|
||||
//! needs only the `beacon` feature, so a service can depend on the SDK with
|
||||
//! `default-features = false, features = ["beacon"]` and pull in none of the
|
||||
//! feeder machinery.
|
||||
|
||||
mod node;
|
||||
mod proto;
|
||||
mod scan;
|
||||
|
||||
pub use node::{enabled_from_env, hostname, BeaconNode, Config, SeenNode, SeenResource};
|
||||
pub use scan::{classify, scan, suggested_name, Candidate, CandidateState, ScanReport, ScanSummary};
|
||||
pub use proto::{
|
||||
cap_matches, rev_of, Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT,
|
||||
LIVENESS_FACTOR, MAX_DATAGRAM, PROTO, VERSION,
|
||||
};
|
||||
|
||||
/// Capability vocabulary of the MetaMesh profile (spec Part 2).
|
||||
pub mod caps {
|
||||
/// meta-core; its resource carries `data.urls`.
|
||||
pub const CORE: &str = "metamesh.core";
|
||||
/// Prefix of `metamesh.service/<name>`.
|
||||
pub const SERVICE_PREFIX: &str = "metamesh.service/";
|
||||
/// Every service.
|
||||
pub const ANY_SERVICE: &str = "metamesh.service/*";
|
||||
/// Prefix of `metamesh.transport/<id>@<contract>`.
|
||||
pub const TRANSPORT_PREFIX: &str = "metamesh.transport/";
|
||||
/// Every transport plugin.
|
||||
pub const ANY_TRANSPORT: &str = "metamesh.transport/*";
|
||||
/// Prefix of `metamesh.feeder/<upstream_id>`.
|
||||
pub const FEEDER_PREFIX: &str = "metamesh.feeder/";
|
||||
/// Every feeder upstream.
|
||||
pub const ANY_FEEDER: &str = "metamesh.feeder/*";
|
||||
/// Prefix of `metamesh.enrich/<plugin_id>` (meta-sort enrichment plugins).
|
||||
pub const ENRICH_PREFIX: &str = "metamesh.enrich/";
|
||||
/// Every meta-sort enrichment plugin.
|
||||
pub const ANY_ENRICH: &str = "metamesh.enrich/*";
|
||||
|
||||
pub fn service(name: &str) -> String {
|
||||
format!("{SERVICE_PREFIX}{name}")
|
||||
}
|
||||
|
||||
pub fn transport(id: &str, contract: u32) -> String {
|
||||
format!("{TRANSPORT_PREFIX}{id}@{contract}")
|
||||
}
|
||||
|
||||
pub fn feeder(upstream_id: &str) -> String {
|
||||
format!("{FEEDER_PREFIX}{upstream_id}")
|
||||
}
|
||||
|
||||
pub fn enrich(plugin_id: &str) -> String {
|
||||
format!("{ENRICH_PREFIX}{plugin_id}")
|
||||
}
|
||||
}
|
||||
|
||||
/// The resource every meta-* service advertises: id `service`, cap
|
||||
/// `metamesh.service/<name>`, `endpoints.ui` = its browser-facing URL.
|
||||
pub fn service_resource(name: &str, ui_url: Option<&str>) -> Resource {
|
||||
let mut r = Resource::new("service", [caps::service(name)]);
|
||||
if let Some(u) = ui_url {
|
||||
r = r.with_endpoint("ui", u);
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
/// `BEACON_ADVERTISE_URL`, else `http://$HOSTNAME:<port>` — what a plugin
|
||||
/// advertises as `endpoints.http`.
|
||||
pub fn advertise_url(listen_port: u16) -> String {
|
||||
std::env::var("BEACON_ADVERTISE_URL")
|
||||
.ok()
|
||||
.map(|s| s.trim().trim_end_matches('/').to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| format!("http://{}:{}", hostname(), listen_port))
|
||||
}
|
||||
|
||||
/// `BEACON_BINDS`, when set.
|
||||
pub fn binds_from_env() -> Option<String> {
|
||||
std::env::var("BEACON_BINDS").ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// Advertise a plugin process: spawn a [`BeaconNode`] named `name` whose single
|
||||
/// resource is whatever `build` returns, and re-run `build` every interval so a
|
||||
/// changed manifest (`rev`) is re-advertised at once. Used by both SDK
|
||||
/// harnesses; returns `None` (and logs) when discovery is disabled or the
|
||||
/// socket can't be set up — a plugin keeps serving either way.
|
||||
pub fn advertise_plugin<F>(name: &str, version: &str, build: F) -> Option<BeaconNode>
|
||||
where
|
||||
F: Fn() -> Resource + Send + Sync + 'static,
|
||||
{
|
||||
if !enabled_from_env() {
|
||||
tracing::info!("beacon: disabled (ENABLE_UDP_DISCOVERY)");
|
||||
return None;
|
||||
}
|
||||
let cfg = Config::from_env(name).version(version).resource(build());
|
||||
let interval = cfg.interval;
|
||||
let node = match BeaconNode::spawn(cfg) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "beacon: could not start; plugin will not be discoverable");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let refresher = node.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(interval);
|
||||
tick.tick().await;
|
||||
loop {
|
||||
tick.tick().await;
|
||||
refresher.set_resources(vec![build()]).await;
|
||||
}
|
||||
});
|
||||
Some(node)
|
||||
}
|
||||
|
||||
/// The `/api/neighbors` JSON every service serves (spec Part 2, "HTTP
|
||||
/// surface"). Each row is a [`SeenNode`] plus `caps` and a v1-compatible
|
||||
/// `baseUrl` (the `metamesh.service/*` resource's `endpoints.ui`).
|
||||
///
|
||||
/// `all` returns every instance instead of one row per name; `cap` keeps only
|
||||
/// nodes owning a resource that matches it.
|
||||
pub fn neighbors_json(
|
||||
node: Option<&BeaconNode>,
|
||||
current: &str,
|
||||
all: bool,
|
||||
cap: Option<&str>,
|
||||
) -> serde_json::Value {
|
||||
let Some(node) = node else {
|
||||
return serde_json::json!({
|
||||
"current": current, "enabled": false, "count": 0, "neighbors": [],
|
||||
});
|
||||
};
|
||||
let rows: Vec<serde_json::Value> = if all { node.nodes() } else { node.nodes_by_name() }
|
||||
.into_iter()
|
||||
.filter(|n| cap.map_or(true, |c| n.matches(c)))
|
||||
.map(|n| neighbor_row(&n))
|
||||
.collect();
|
||||
serde_json::json!({
|
||||
"current": current,
|
||||
"enabled": true,
|
||||
"count": rows.len(),
|
||||
"self": neighbor_row(&node.self_node()),
|
||||
"neighbors": rows,
|
||||
})
|
||||
}
|
||||
|
||||
fn neighbor_row(n: &SeenNode) -> serde_json::Value {
|
||||
let mut v = serde_json::to_value(n).unwrap_or_default();
|
||||
if let Some(obj) = v.as_object_mut() {
|
||||
obj.insert("caps".into(), serde_json::json!(n.caps()));
|
||||
let base = n.resource(caps::ANY_SERVICE).and_then(|r| r.endpoint("ui"));
|
||||
if let Some(b) = base {
|
||||
obj.insert("baseUrl".into(), serde_json::Value::String(b));
|
||||
}
|
||||
}
|
||||
v
|
||||
}
|
||||
@@ -0,0 +1,719 @@
|
||||
//! `BeaconNode` — advertise this process's resources and keep a live view of
|
||||
//! everyone else's.
|
||||
//!
|
||||
//! Ported from meta-discovery v1's `meshdisco.rs`, keeping the two details that
|
||||
//! fail *silently* when wrong:
|
||||
//!
|
||||
//! 1. **Per-interface fan-out.** A socket that joins with `INADDR_ANY` and sends
|
||||
//! via the default route covers exactly one interface. `metashare-app` sits
|
||||
//! on `pcs` *and* `metamesh-mesh`; `metawatch-*` on `metawatch` *and*
|
||||
//! `metamesh-mesh`. We enumerate interfaces and join/send on each, using
|
||||
//! `ip_mreqn`'s `imr_ifindex`.
|
||||
//! 2. **Expiry at read time.** No reaper owns correctness; the announce loop's
|
||||
//! prune only exists to fire change notifications for nodes that went quiet.
|
||||
//!
|
||||
//! Raw `libc` rather than `socket2` / `if-addrs`: every consumer already links
|
||||
//! libc, and this is the whole of what we need from either crate.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
|
||||
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::Serialize;
|
||||
use tokio::net::UdpSocket;
|
||||
use tokio::sync::watch;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use super::proto::{
|
||||
Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT, LIVENESS_FACTOR,
|
||||
MAX_DATAGRAM, READ_BUFFER, TYPE_ADVERTISE, TYPE_BYE, TYPE_PROBE,
|
||||
};
|
||||
|
||||
/// How a node is set up. Build with [`Config::new`] / [`Config::from_env`].
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Config {
|
||||
pub node: NodeInfo,
|
||||
pub resources: Vec<Resource>,
|
||||
pub group: String,
|
||||
pub port: u16,
|
||||
pub interval: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Defaults: this host's name as the instance, the standard group/port,
|
||||
/// a 10 s interval, no resources.
|
||||
pub fn new(name: impl Into<String>) -> Self {
|
||||
Self {
|
||||
node: NodeInfo {
|
||||
name: name.into(),
|
||||
instance: hostname(),
|
||||
version: None,
|
||||
status: None,
|
||||
},
|
||||
resources: Vec::new(),
|
||||
group: DEFAULT_GROUP.into(),
|
||||
port: DEFAULT_PORT,
|
||||
interval: DEFAULT_INTERVAL,
|
||||
}
|
||||
}
|
||||
|
||||
/// [`Config::new`] with `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS`
|
||||
/// applied.
|
||||
pub fn from_env(name: impl Into<String>) -> Self {
|
||||
let mut c = Self::new(name);
|
||||
if let Some(g) = env_nonempty("BEACON_GROUP") {
|
||||
c.group = g;
|
||||
}
|
||||
if let Some(p) = env_nonempty("BEACON_PORT").and_then(|v| v.parse().ok()) {
|
||||
c.port = p;
|
||||
}
|
||||
if let Some(ms) = env_nonempty("BEACON_INTERVAL_MS").and_then(|v| v.parse::<u64>().ok()) {
|
||||
if ms > 0 {
|
||||
c.interval = Duration::from_millis(ms);
|
||||
}
|
||||
}
|
||||
c
|
||||
}
|
||||
|
||||
pub fn version(mut self, v: impl Into<String>) -> Self {
|
||||
self.node.version = Some(v.into());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn instance(mut self, i: impl Into<String>) -> Self {
|
||||
self.node.instance = i.into();
|
||||
self
|
||||
}
|
||||
|
||||
pub fn resource(mut self, r: Resource) -> Self {
|
||||
self.resources.push(r);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A node heard on the wire (or this node itself, via [`BeaconNode::self_node`]).
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SeenNode {
|
||||
#[serde(flatten)]
|
||||
pub node: NodeInfo,
|
||||
pub resources: Vec<Resource>,
|
||||
/// Source address taken from the packet, never from the payload.
|
||||
pub addr: String,
|
||||
/// Unix seconds.
|
||||
#[serde(rename = "lastSeen")]
|
||||
pub last_seen: f64,
|
||||
}
|
||||
|
||||
impl SeenNode {
|
||||
/// Every cap of every resource, in order, deduplicated.
|
||||
pub fn caps(&self) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for c in self.resources.iter().flat_map(|r| r.caps.iter()) {
|
||||
if !out.contains(c) {
|
||||
out.push(c.clone());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// True if any resource matches `pattern`.
|
||||
pub fn matches(&self, pattern: &str) -> bool {
|
||||
self.resources.iter().any(|r| r.matches(pattern))
|
||||
}
|
||||
|
||||
/// The first resource matching `pattern`.
|
||||
pub fn resource(&self, pattern: &str) -> Option<&Resource> {
|
||||
self.resources.iter().find(|r| r.matches(pattern))
|
||||
}
|
||||
}
|
||||
|
||||
/// One resource together with the node advertising it.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct SeenResource {
|
||||
pub node: NodeInfo,
|
||||
pub resource: Resource,
|
||||
pub addr: String,
|
||||
#[serde(rename = "lastSeen")]
|
||||
pub last_seen: f64,
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
group: Ipv4Addr,
|
||||
port: u16,
|
||||
interval: Duration,
|
||||
node: RwLock<NodeInfo>,
|
||||
resources: RwLock<Vec<Resource>>,
|
||||
socket: UdpSocket,
|
||||
ifindexes: Vec<(String, u32)>,
|
||||
seen: Mutex<HashMap<String, SeenNode>>,
|
||||
changes: watch::Sender<u64>,
|
||||
stopped: AtomicBool,
|
||||
}
|
||||
|
||||
/// A running beacon v2 participant. Cheap to clone.
|
||||
#[derive(Clone)]
|
||||
pub struct BeaconNode {
|
||||
inner: Arc<Inner>,
|
||||
}
|
||||
|
||||
impl BeaconNode {
|
||||
/// Bind, join the group on every eligible interface, and start the read and
|
||||
/// advertise loops. Must be called inside a tokio runtime.
|
||||
///
|
||||
/// A join failure on one interface is logged and skipped rather than fatal —
|
||||
/// a host that blocks multicast must degrade, not fail to boot.
|
||||
pub fn spawn(cfg: Config) -> io::Result<BeaconNode> {
|
||||
let group: Ipv4Addr = cfg.group.parse().map_err(|_| {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, format!("bad group {}", cfg.group))
|
||||
})?;
|
||||
|
||||
let std_sock = bind_reuse(cfg.port)?;
|
||||
set_multicast_ttl(&std_sock, 1)?;
|
||||
|
||||
let ifindexes = eligible_interfaces();
|
||||
let mut joined = 0usize;
|
||||
for (name, idx) in &ifindexes {
|
||||
match join_group(&std_sock, group, *idx) {
|
||||
Ok(()) => joined += 1,
|
||||
Err(e) => warn!(iface = %name, error = %e, "beacon: join failed"),
|
||||
}
|
||||
}
|
||||
info!(
|
||||
group = %cfg.group, port = cfg.port, name = %cfg.node.name,
|
||||
instance = %cfg.node.instance, resources = cfg.resources.len(),
|
||||
joined, total = ifindexes.len(),
|
||||
"beacon: listening"
|
||||
);
|
||||
|
||||
std_sock.set_nonblocking(true)?;
|
||||
let socket = UdpSocket::from_std(std_sock)?;
|
||||
let (changes, _) = watch::channel(0u64);
|
||||
|
||||
let node = BeaconNode {
|
||||
inner: Arc::new(Inner {
|
||||
group,
|
||||
port: cfg.port,
|
||||
interval: cfg.interval,
|
||||
node: RwLock::new(cfg.node),
|
||||
resources: RwLock::new(cfg.resources),
|
||||
socket,
|
||||
ifindexes,
|
||||
seen: Mutex::new(HashMap::new()),
|
||||
changes,
|
||||
stopped: AtomicBool::new(false),
|
||||
}),
|
||||
};
|
||||
|
||||
let reader = node.clone();
|
||||
tokio::spawn(async move { reader.read_loop().await });
|
||||
|
||||
let advertiser = node.clone();
|
||||
tokio::spawn(async move {
|
||||
advertiser.advertise().await;
|
||||
advertiser.probe(&[]).await;
|
||||
let mut tick = tokio::time::interval(advertiser.inner.interval);
|
||||
tick.tick().await; // the immediate first tick
|
||||
loop {
|
||||
tick.tick().await;
|
||||
if advertiser.inner.stopped.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
advertiser.advertise().await;
|
||||
advertiser.prune();
|
||||
}
|
||||
});
|
||||
|
||||
Ok(node)
|
||||
}
|
||||
|
||||
/// The instance name this node advertises as.
|
||||
pub fn instance(&self) -> String {
|
||||
self.inner.node.read().unwrap().instance.clone()
|
||||
}
|
||||
|
||||
/// Replace this node's resources and re-advertise at once (e.g. a manifest
|
||||
/// `rev` changed). A no-op if nothing changed.
|
||||
pub async fn set_resources(&self, resources: Vec<Resource>) {
|
||||
{
|
||||
let mut cur = self.inner.resources.write().unwrap();
|
||||
if *cur == resources {
|
||||
return;
|
||||
}
|
||||
*cur = resources;
|
||||
}
|
||||
self.advertise().await;
|
||||
}
|
||||
|
||||
/// Set `node.status` (`starting` / `running`) and re-advertise.
|
||||
pub async fn set_status(&self, status: &str) {
|
||||
{
|
||||
let mut n = self.inner.node.write().unwrap();
|
||||
if n.status.as_deref() == Some(status) {
|
||||
return;
|
||||
}
|
||||
n.status = Some(status.to_string());
|
||||
}
|
||||
self.advertise().await;
|
||||
}
|
||||
|
||||
/// Multicast a probe. Every node owning a resource matching one of `want`
|
||||
/// (every node, when `want` is empty) replies at once — what makes the
|
||||
/// protocol usable on a boot path instead of waiting out an interval.
|
||||
pub async fn probe(&self, want: &[String]) {
|
||||
let msg = Message::probe(&self.instance(), want);
|
||||
self.multicast(&msg).await;
|
||||
}
|
||||
|
||||
/// Say goodbye: multicast `bye` and stop advertising. Call on clean
|
||||
/// shutdown so neighbours drop us at once instead of after the TTL.
|
||||
pub async fn bye(&self) {
|
||||
if self.inner.stopped.swap(true, Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let node = self.inner.node.read().unwrap().clone();
|
||||
self.multicast(&Message::bye(node)).await;
|
||||
}
|
||||
|
||||
/// Fires (the counter increments) whenever the set of live nodes or any
|
||||
/// node's resources change — new node, changed advertise, bye, expiry.
|
||||
/// Consumers re-read [`BeaconNode::resources`] on each change.
|
||||
pub fn subscribe(&self) -> watch::Receiver<u64> {
|
||||
self.inner.changes.subscribe()
|
||||
}
|
||||
|
||||
/// Every live node, sorted by (name, instance).
|
||||
pub fn nodes(&self) -> Vec<SeenNode> {
|
||||
let cutoff = self.cutoff();
|
||||
let guard = self.inner.seen.lock().unwrap();
|
||||
let mut out: Vec<SeenNode> = guard.values().filter(|n| n.last_seen >= cutoff).cloned().collect();
|
||||
drop(guard);
|
||||
out.sort_by(|a, b| a.node.name.cmp(&b.node.name).then(a.node.instance.cmp(&b.node.instance)));
|
||||
out
|
||||
}
|
||||
|
||||
/// One row per node name, most recently seen instance wins — the nav
|
||||
/// menu's shape.
|
||||
pub fn nodes_by_name(&self) -> Vec<SeenNode> {
|
||||
let mut best: HashMap<String, SeenNode> = HashMap::new();
|
||||
for n in self.nodes() {
|
||||
match best.get(&n.node.name) {
|
||||
Some(cur) if cur.last_seen >= n.last_seen => {}
|
||||
_ => {
|
||||
best.insert(n.node.name.clone(), n);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out: Vec<SeenNode> = best.into_values().collect();
|
||||
out.sort_by(|a, b| a.node.name.cmp(&b.node.name));
|
||||
out
|
||||
}
|
||||
|
||||
/// Every live resource matching `pattern`, with its node.
|
||||
pub fn resources(&self, pattern: &str) -> Vec<SeenResource> {
|
||||
self.nodes()
|
||||
.into_iter()
|
||||
.flat_map(|n| {
|
||||
let SeenNode { node, resources, addr, last_seen } = n;
|
||||
resources.into_iter().filter(|r| r.matches(pattern)).map(move |resource| SeenResource {
|
||||
node: node.clone(),
|
||||
resource,
|
||||
addr: addr.clone(),
|
||||
last_seen,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// This node as a row, so a UI can render itself without waiting for its
|
||||
/// own echo.
|
||||
pub fn self_node(&self) -> SeenNode {
|
||||
SeenNode {
|
||||
node: self.inner.node.read().unwrap().clone(),
|
||||
resources: self.inner.resources.read().unwrap().clone(),
|
||||
addr: String::new(),
|
||||
last_seen: now_secs(),
|
||||
}
|
||||
}
|
||||
|
||||
// -- internals ----------------------------------------------------------
|
||||
|
||||
fn cutoff(&self) -> f64 {
|
||||
now_secs() - self.inner.interval.as_secs_f64() * LIVENESS_FACTOR as f64
|
||||
}
|
||||
|
||||
fn bump(&self) {
|
||||
self.inner.changes.send_modify(|g| *g = g.wrapping_add(1));
|
||||
}
|
||||
|
||||
/// Drop expired nodes; notify if any went.
|
||||
fn prune(&self) {
|
||||
let cutoff = self.cutoff();
|
||||
let removed = {
|
||||
let mut guard = self.inner.seen.lock().unwrap();
|
||||
let before = guard.len();
|
||||
guard.retain(|_, n| n.last_seen >= cutoff);
|
||||
before != guard.len()
|
||||
};
|
||||
if removed {
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
|
||||
fn advertise_msg(&self) -> Message {
|
||||
Message::advertise(
|
||||
self.inner.node.read().unwrap().clone(),
|
||||
self.inner.resources.read().unwrap().clone(),
|
||||
)
|
||||
}
|
||||
|
||||
async fn advertise(&self) {
|
||||
if self.inner.stopped.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
let msg = self.advertise_msg();
|
||||
self.multicast(&msg).await;
|
||||
}
|
||||
|
||||
fn encode(msg: &Message) -> Option<Vec<u8>> {
|
||||
match serde_json::to_vec(msg) {
|
||||
Ok(b) => {
|
||||
if b.len() > MAX_DATAGRAM {
|
||||
warn!(bytes = b.len(), max = MAX_DATAGRAM, "beacon: datagram over the fragmentation-safe size");
|
||||
}
|
||||
Some(b)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(error = %e, "beacon: encode failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes once per interface. Letting the routing table choose would reach
|
||||
/// exactly one network — the failure this protocol exists to avoid.
|
||||
async fn multicast(&self, msg: &Message) {
|
||||
let Some(body) = Self::encode(msg) else { return };
|
||||
let dst = SocketAddr::V4(SocketAddrV4::new(self.inner.group, self.inner.port));
|
||||
for (name, idx) in &self.inner.ifindexes {
|
||||
if let Err(e) = set_multicast_if(&self.inner.socket, *idx) {
|
||||
warn!(iface = %name, error = %e, "beacon: select interface failed");
|
||||
continue;
|
||||
}
|
||||
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
|
||||
warn!(iface = %name, error = %e, "beacon: send failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn reply_to(&self, dst: SocketAddr) {
|
||||
let Some(body) = Self::encode(&self.advertise_msg()) else { return };
|
||||
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
|
||||
debug!(error = %e, "beacon: reply failed");
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_loop(self) {
|
||||
let mut buf = vec![0u8; READ_BUFFER];
|
||||
loop {
|
||||
let (n, src) = match self.inner.socket.recv_from(&mut buf).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
debug!(error = %e, "beacon: read error");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let Some(msg) = Message::parse(&buf[..n]) else { continue };
|
||||
let me = self.instance();
|
||||
|
||||
match msg.msg_type.as_str() {
|
||||
TYPE_PROBE => {
|
||||
if msg.from.as_deref() == Some(me.as_str()) || self.inner.stopped.load(Ordering::Relaxed) {
|
||||
continue;
|
||||
}
|
||||
let wanted = msg.want.is_empty()
|
||||
|| self.inner.resources.read().unwrap().iter().any(|r| r.matches_any(&msg.want));
|
||||
if wanted {
|
||||
self.reply_to(src).await;
|
||||
}
|
||||
}
|
||||
TYPE_ADVERTISE | TYPE_BYE => {
|
||||
let Some(node) = msg.node else { continue };
|
||||
if node.instance == me {
|
||||
continue; // our own multicast echo
|
||||
}
|
||||
let changed = if msg.msg_type == TYPE_BYE {
|
||||
self.inner.seen.lock().unwrap().remove(&node.instance).is_some()
|
||||
} else {
|
||||
let cutoff = self.cutoff();
|
||||
let mut guard = self.inner.seen.lock().unwrap();
|
||||
let changed = match guard.get(&node.instance) {
|
||||
Some(prev) => {
|
||||
prev.last_seen < cutoff || prev.node != node || prev.resources != msg.resources
|
||||
}
|
||||
None => true,
|
||||
};
|
||||
guard.insert(
|
||||
node.instance.clone(),
|
||||
SeenNode {
|
||||
node,
|
||||
resources: msg.resources,
|
||||
addr: src.ip().to_string(),
|
||||
last_seen: now_secs(),
|
||||
},
|
||||
);
|
||||
changed
|
||||
};
|
||||
if changed {
|
||||
self.bump();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Socket plumbing.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn bind_reuse(port: u16) -> io::Result<std::net::UdpSocket> {
|
||||
unsafe {
|
||||
let fd: RawFd = libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0);
|
||||
if fd < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
// Must be set before bind. Lets several listeners share the port on one
|
||||
// host (a test, a sidecar, a beacon v1 responder in the same netns).
|
||||
let one: libc::c_int = 1;
|
||||
for opt in [libc::SO_REUSEADDR, libc::SO_REUSEPORT] {
|
||||
libc::setsockopt(
|
||||
fd,
|
||||
libc::SOL_SOCKET,
|
||||
opt,
|
||||
&one as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
);
|
||||
}
|
||||
let addr = libc::sockaddr_in {
|
||||
sin_family: libc::AF_INET as libc::sa_family_t,
|
||||
sin_port: port.to_be(),
|
||||
sin_addr: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
|
||||
sin_zero: [0; 8],
|
||||
};
|
||||
if libc::bind(
|
||||
fd,
|
||||
&addr as *const _ as *const libc::sockaddr,
|
||||
std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
|
||||
) < 0
|
||||
{
|
||||
let err = io::Error::last_os_error();
|
||||
libc::close(fd);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(std::net::UdpSocket::from_raw_fd(fd))
|
||||
}
|
||||
}
|
||||
|
||||
fn set_multicast_ttl(sock: &std::net::UdpSocket, ttl: libc::c_int) -> io::Result<()> {
|
||||
// TTL 1: link-local only, never routed off the segment.
|
||||
let rc = unsafe {
|
||||
libc::setsockopt(
|
||||
sock.as_raw_fd(),
|
||||
libc::IPPROTO_IP,
|
||||
libc::IP_MULTICAST_TTL,
|
||||
&ttl as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if rc < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `ip_mreqn` — `imr_ifindex` is what makes this a per-interface join instead
|
||||
/// of "whatever the routing table picks".
|
||||
fn mreqn(group: Ipv4Addr, ifindex: u32) -> libc::ip_mreqn {
|
||||
libc::ip_mreqn {
|
||||
imr_multiaddr: libc::in_addr { s_addr: u32::from(group).to_be() },
|
||||
imr_address: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
|
||||
imr_ifindex: ifindex as libc::c_int,
|
||||
}
|
||||
}
|
||||
|
||||
fn join_group(sock: &std::net::UdpSocket, group: Ipv4Addr, ifindex: u32) -> io::Result<()> {
|
||||
let req = mreqn(group, ifindex);
|
||||
let rc = unsafe {
|
||||
libc::setsockopt(
|
||||
sock.as_raw_fd(),
|
||||
libc::IPPROTO_IP,
|
||||
libc::IP_ADD_MEMBERSHIP,
|
||||
&req as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if rc < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_multicast_if(sock: &UdpSocket, ifindex: u32) -> io::Result<()> {
|
||||
let req = mreqn(Ipv4Addr::UNSPECIFIED, ifindex);
|
||||
let rc = unsafe {
|
||||
libc::setsockopt(
|
||||
sock.as_raw_fd(),
|
||||
libc::IPPROTO_IP,
|
||||
libc::IP_MULTICAST_IF,
|
||||
&req as *const _ as *const libc::c_void,
|
||||
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
|
||||
)
|
||||
};
|
||||
if rc < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Every interface except loopback, as (name, ifindex).
|
||||
fn eligible_interfaces() -> Vec<(String, u32)> {
|
||||
let mut out = Vec::new();
|
||||
unsafe {
|
||||
let list = libc::if_nameindex();
|
||||
if list.is_null() {
|
||||
warn!("beacon: could not enumerate interfaces");
|
||||
return out;
|
||||
}
|
||||
let mut p = list;
|
||||
while (*p).if_index != 0 && !(*p).if_name.is_null() {
|
||||
let name = std::ffi::CStr::from_ptr((*p).if_name).to_string_lossy().into_owned();
|
||||
if name != "lo" {
|
||||
out.push((name, (*p).if_index));
|
||||
}
|
||||
p = p.add(1);
|
||||
}
|
||||
libc::if_freenameindex(list);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn env_nonempty(key: &str) -> Option<String> {
|
||||
std::env::var(key).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// `$HOSTNAME`, else `/etc/hostname`, else `unknown`.
|
||||
pub fn hostname() -> String {
|
||||
env_nonempty("HOSTNAME")
|
||||
.or_else(|| {
|
||||
std::fs::read_to_string("/etc/hostname")
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
})
|
||||
.unwrap_or_else(|| "unknown".into())
|
||||
}
|
||||
|
||||
pub(crate) fn now_secs() -> f64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0)
|
||||
}
|
||||
|
||||
/// `ENABLE_UDP_DISCOVERY`, permissive: only `false`/`0`/`no`/`off` disable.
|
||||
pub fn enabled_from_env() -> bool {
|
||||
match std::env::var("ENABLE_UDP_DISCOVERY") {
|
||||
Ok(v) => !matches!(v.trim().to_ascii_lowercase().as_str(), "false" | "0" | "no" | "off"),
|
||||
Err(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Two nodes on loopback-only hosts can't multicast to each other, so
|
||||
/// exercise the receive path directly by unicasting into a node's port.
|
||||
async fn node_on(port: u16, instance: &str, resources: Vec<Resource>) -> BeaconNode {
|
||||
let mut cfg = Config::new("test").instance(instance);
|
||||
cfg.port = port;
|
||||
cfg.interval = Duration::from_millis(200);
|
||||
cfg.resources = resources;
|
||||
BeaconNode::spawn(cfg).unwrap()
|
||||
}
|
||||
|
||||
async fn send(port: u16, msg: &Message) {
|
||||
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
|
||||
s.send_to(&serde_json::to_vec(msg).unwrap(), ("127.0.0.1", port)).await.unwrap();
|
||||
}
|
||||
|
||||
fn free_port() -> u16 {
|
||||
std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
|
||||
}
|
||||
|
||||
fn info(instance: &str) -> NodeInfo {
|
||||
NodeInfo { name: "peer".into(), instance: instance.into(), ..Default::default() }
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn advertise_bye_and_expiry() {
|
||||
let port = free_port();
|
||||
let node = node_on(port, "me", vec![]).await;
|
||||
let mut rx = node.subscribe();
|
||||
|
||||
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
|
||||
send(port, &Message::advertise(info("peer-1"), vec![r])).await;
|
||||
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
|
||||
assert_eq!(node.resources("metamesh.transport/*").len(), 1);
|
||||
assert!(node.resources("metamesh.core").is_empty());
|
||||
|
||||
send(port, &Message::bye(info("peer-1"))).await;
|
||||
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
|
||||
assert!(node.nodes().is_empty());
|
||||
|
||||
send(port, &Message::advertise(info("peer-2"), vec![])).await;
|
||||
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
|
||||
assert_eq!(node.nodes().len(), 1);
|
||||
// 3 × 200 ms liveness, then the prune on the next tick notifies.
|
||||
tokio::time::timeout(Duration::from_secs(3), rx.changed()).await.unwrap().unwrap();
|
||||
assert!(node.nodes().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_echo_and_v1_ignored() {
|
||||
let port = free_port();
|
||||
let node = node_on(port, "me", vec![]).await;
|
||||
send(port, &Message::advertise(info("me"), vec![])).await;
|
||||
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
|
||||
s.send_to(br#"{"type":"announce","name":"x","tools":[]}"#, ("127.0.0.1", port)).await.unwrap();
|
||||
s.send_to(br#"{"v":1,"type":"announce","name":"meta-sort","instance":"a"}"#, ("127.0.0.1", port))
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
assert!(node.nodes().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_reply_honours_want() {
|
||||
let port = free_port();
|
||||
let _node = node_on(port, "me", vec![Resource::new("service", ["metamesh.service/meta-sort"])]).await;
|
||||
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
|
||||
let mut buf = vec![0u8; READ_BUFFER];
|
||||
|
||||
let miss = Message::probe("cli", &["metamesh.core".to_string()]);
|
||||
s.send_to(&serde_json::to_vec(&miss).unwrap(), ("127.0.0.1", port)).await.unwrap();
|
||||
assert!(tokio::time::timeout(Duration::from_millis(300), s.recv_from(&mut buf)).await.is_err());
|
||||
|
||||
let hit = Message::probe("cli", &["metamesh.service/*".to_string()]);
|
||||
s.send_to(&serde_json::to_vec(&hit).unwrap(), ("127.0.0.1", port)).await.unwrap();
|
||||
let (n, _) = tokio::time::timeout(Duration::from_secs(2), s.recv_from(&mut buf)).await.unwrap().unwrap();
|
||||
let m = Message::parse(&buf[..n]).unwrap();
|
||||
assert_eq!(m.msg_type, TYPE_ADVERTISE);
|
||||
assert_eq!(m.node.unwrap().instance, "me");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
//! Beacon v2 wire types and capability matching.
|
||||
//!
|
||||
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const PROTO: &str = "beacon";
|
||||
pub const VERSION: u8 = 2;
|
||||
/// Beacon v1's group and port, on purpose — one well-known endpoint for every
|
||||
/// kind of local resource. v1 and v2 coexist because v2 never sends v1's
|
||||
/// `discovery` / `announce` types.
|
||||
pub const DEFAULT_GROUP: &str = "239.255.99.1";
|
||||
pub const DEFAULT_PORT: u16 = 9099;
|
||||
pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(10);
|
||||
/// A node unheard for `interval * this` is dropped (evaluated at read time).
|
||||
pub const LIVENESS_FACTOR: u32 = 3;
|
||||
/// Receive buffer. Senders stay under [`MAX_DATAGRAM`].
|
||||
pub const READ_BUFFER: usize = 8192;
|
||||
/// Above this a datagram risks IP fragmentation; the sender logs a warning.
|
||||
pub const MAX_DATAGRAM: usize = 1400;
|
||||
|
||||
pub const TYPE_PROBE: &str = "probe";
|
||||
pub const TYPE_ADVERTISE: &str = "advertise";
|
||||
pub const TYPE_BYE: &str = "bye";
|
||||
|
||||
/// Who is advertising.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct NodeInfo {
|
||||
/// What the software is (`meta-sort`, `meta-transport-nzb`).
|
||||
pub name: String,
|
||||
/// Unique per running copy — the container hostname.
|
||||
pub instance: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub version: Option<String>,
|
||||
/// `starting` | `running`. Absent reads as `running`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
}
|
||||
|
||||
/// One advertised resource.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
|
||||
pub struct Resource {
|
||||
/// Unique within the node.
|
||||
pub id: String,
|
||||
pub caps: Vec<String>,
|
||||
/// Well-known names: `http`, `ui`, `manifest`, `mcp`. A value starting with
|
||||
/// `/` is relative to `http` — see [`Resource::endpoint`].
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub endpoints: BTreeMap<String, String>,
|
||||
/// Opaque revision of the resource's details (e.g. its manifest).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub rev: Option<String>,
|
||||
/// The one consumer instance this resource belongs to. Absent = anyone.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub binds: Option<String>,
|
||||
/// Profile-defined, per capability.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub data: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Resource {
|
||||
pub fn new(id: impl Into<String>, caps: impl IntoIterator<Item = impl Into<String>>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
caps: caps.into_iter().map(Into::into).collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_endpoint(mut self, name: impl Into<String>, url: impl Into<String>) -> Self {
|
||||
let url = url.into();
|
||||
if !url.is_empty() {
|
||||
self.endpoints.insert(name.into(), url);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// True if any of this resource's caps matches `pattern`.
|
||||
pub fn matches(&self, pattern: &str) -> bool {
|
||||
self.caps.iter().any(|c| cap_matches(pattern, c))
|
||||
}
|
||||
|
||||
/// True if any cap matches any of `patterns`. An empty list matches.
|
||||
pub fn matches_any(&self, patterns: &[String]) -> bool {
|
||||
patterns.is_empty() || patterns.iter().any(|p| self.matches(p))
|
||||
}
|
||||
|
||||
/// `binds` is absent, or names `instance`.
|
||||
pub fn usable_by(&self, instance: &str) -> bool {
|
||||
self.binds.as_deref().map_or(true, |b| b == instance)
|
||||
}
|
||||
|
||||
/// The named endpoint as an absolute URL: a value starting with `/` is
|
||||
/// joined onto `endpoints.http`.
|
||||
pub fn endpoint(&self, name: &str) -> Option<String> {
|
||||
let v = self.endpoints.get(name)?;
|
||||
if v.starts_with('/') {
|
||||
let base = self.endpoints.get("http")?;
|
||||
Some(format!("{}{}", base.trim_end_matches('/'), v))
|
||||
} else {
|
||||
Some(v.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every v2 datagram.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Message {
|
||||
pub proto: String,
|
||||
pub v: u8,
|
||||
#[serde(rename = "type")]
|
||||
pub msg_type: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub node: Option<NodeInfo>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub resources: Vec<Resource>,
|
||||
/// Probe only: informational sender instance.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub from: Option<String>,
|
||||
/// Probe only: reply only if a resource matches one of these.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub want: Vec<String>,
|
||||
}
|
||||
|
||||
impl Message {
|
||||
fn base(msg_type: &str) -> Self {
|
||||
Self {
|
||||
proto: PROTO.into(),
|
||||
v: VERSION,
|
||||
msg_type: msg_type.into(),
|
||||
node: None,
|
||||
resources: Vec::new(),
|
||||
from: None,
|
||||
want: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn probe(from: &str, want: &[String]) -> Self {
|
||||
Self {
|
||||
from: Some(from.to_string()),
|
||||
want: want.to_vec(),
|
||||
..Self::base(TYPE_PROBE)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn advertise(node: NodeInfo, resources: Vec<Resource>) -> Self {
|
||||
Self {
|
||||
node: Some(node),
|
||||
resources,
|
||||
..Self::base(TYPE_ADVERTISE)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bye(node: NodeInfo) -> Self {
|
||||
Self {
|
||||
node: Some(node),
|
||||
..Self::base(TYPE_BYE)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a datagram, returning `None` for anything that is not a
|
||||
/// well-formed beacon v2 message — beacon v1 traffic (no envelope), other
|
||||
/// versions, unknown types, or an advertise/bye without a node. The group
|
||||
/// is shared, so all of that is silently ignored.
|
||||
pub fn parse(bytes: &[u8]) -> Option<Self> {
|
||||
let m: Message = serde_json::from_slice(bytes).ok()?;
|
||||
if m.proto != PROTO || m.v != VERSION {
|
||||
return None;
|
||||
}
|
||||
match m.msg_type.as_str() {
|
||||
TYPE_PROBE => Some(m),
|
||||
TYPE_ADVERTISE | TYPE_BYE => {
|
||||
let n = m.node.as_ref()?;
|
||||
if n.instance.is_empty() || n.name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(m)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Split `base@contract`. A non-numeric or missing contract is `None`.
|
||||
fn split_contract(s: &str) -> (&str, Option<&str>) {
|
||||
match s.rsplit_once('@') {
|
||||
Some((base, c)) => (base, Some(c)),
|
||||
None => (s, None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Does capability `cap` satisfy `pattern`?
|
||||
///
|
||||
/// - `*` matches everything.
|
||||
/// - `@N` on the pattern must equal the cap's contract; without one the cap's
|
||||
/// contract is ignored.
|
||||
/// - `x/*` matches any variant of `x`; otherwise the bases must be equal.
|
||||
pub fn cap_matches(pattern: &str, cap: &str) -> bool {
|
||||
if pattern == "*" {
|
||||
return true;
|
||||
}
|
||||
let (pbase, pcontract) = split_contract(pattern);
|
||||
let (cbase, ccontract) = split_contract(cap);
|
||||
if let Some(pc) = pcontract {
|
||||
if ccontract != Some(pc) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
match pbase.strip_suffix("/*") {
|
||||
Some(prefix) => cbase
|
||||
.strip_prefix(prefix)
|
||||
.is_some_and(|rest| rest.starts_with('/') && rest.len() > 1),
|
||||
None => pbase == cbase,
|
||||
}
|
||||
}
|
||||
|
||||
/// A short opaque revision for any serialisable value — what a harness puts in
|
||||
/// [`Resource::rev`] so consumers re-fetch a manifest only when it changed.
|
||||
/// Stable for a given build; not cryptographic and not meant to be.
|
||||
pub fn rev_of<T: Serialize>(value: &T) -> String {
|
||||
let bytes = serde_json::to_vec(value).unwrap_or_default();
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
bytes.hash(&mut h);
|
||||
format!("{:016x}", h.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn wildcard_matches_any_variant_only() {
|
||||
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/nzb@1"));
|
||||
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/ipfs"));
|
||||
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport"));
|
||||
assert!(!cap_matches("metamesh.transport/*", "metamesh.transportx/a"));
|
||||
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport/"));
|
||||
assert!(cap_matches("*", "anything@3"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn contract_rules() {
|
||||
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@1"));
|
||||
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@2"));
|
||||
assert!(cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@2"));
|
||||
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@1"));
|
||||
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb"));
|
||||
assert!(cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@1"));
|
||||
assert!(!cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_match() {
|
||||
assert!(cap_matches("mcp", "mcp"));
|
||||
assert!(!cap_matches("mcp", "mcp/x"));
|
||||
assert!(!cap_matches("metamesh.core", "metamesh.core2"));
|
||||
assert!(!cap_matches("metamesh.transport/nzb", "metamesh.transport/ipfs@1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_v1_and_foreign_traffic() {
|
||||
// beacon v1 aggregator probe / server announce
|
||||
assert!(Message::parse(br#"{"type":"discovery"}"#).is_none());
|
||||
assert!(Message::parse(br#"{"type":"announce","name":"x","tools":[]}"#).is_none());
|
||||
// meta-discovery v1
|
||||
assert!(Message::parse(br#"{"v":1,"type":"announce","name":"meta-sort"}"#).is_none());
|
||||
// wrong version / unknown type / not json
|
||||
assert!(Message::parse(br#"{"proto":"beacon","v":3,"type":"probe"}"#).is_none());
|
||||
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"announce"}"#).is_none());
|
||||
assert!(Message::parse(b"\x00\x01garbage").is_none());
|
||||
// advertise without a node
|
||||
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"advertise"}"#).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_advertise() {
|
||||
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"])
|
||||
.with_endpoint("http", "http://metashare-nzb:3000")
|
||||
.with_endpoint("manifest", "/manifest");
|
||||
let m = Message::advertise(
|
||||
NodeInfo { name: "meta-transport-nzb".into(), instance: "metashare-nzb".into(), ..Default::default() },
|
||||
vec![r],
|
||||
);
|
||||
let bytes = serde_json::to_vec(&m).unwrap();
|
||||
let back = Message::parse(&bytes).unwrap();
|
||||
assert_eq!(back.msg_type, TYPE_ADVERTISE);
|
||||
let r = &back.resources[0];
|
||||
assert_eq!(r.endpoint("manifest").as_deref(), Some("http://metashare-nzb:3000/manifest"));
|
||||
assert!(r.matches("metamesh.transport/*"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binds() {
|
||||
let mut r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
|
||||
assert!(r.usable_by("anyone"));
|
||||
r.binds = Some("metashare-app".into());
|
||||
assert!(r.usable_by("metashare-app"));
|
||||
assert!(!r.usable_by("metawatch-share"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn probe_filter() {
|
||||
let r = Resource::new("service", ["metamesh.service/meta-sort"]);
|
||||
assert!(r.matches_any(&[]));
|
||||
assert!(r.matches_any(&["metamesh.service/*".into()]));
|
||||
assert!(!r.matches_any(&["metamesh.core".into()]));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
//! On-demand discovery scan — what a consumer's "Scan" button runs.
|
||||
//!
|
||||
//! Probe everyone, wait a moment for the replies, then classify every resource
|
||||
//! matching the consumer's capability pattern against what the consumer has
|
||||
//! already configured. The report is the same JSON in every consumer
|
||||
//! (meta-share, meta-gateway, meta-sort), rendered by the shared
|
||||
//! `<meta-beacon-scan>` element.
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use super::node::{BeaconNode, SeenResource};
|
||||
|
||||
/// How a discovered resource relates to this consumer.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum CandidateState {
|
||||
/// Its URL is already in the consumer's list.
|
||||
Configured,
|
||||
/// Can be added.
|
||||
Addable,
|
||||
/// `binds` names another consumer; shown but not addable.
|
||||
BoundElsewhere,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Candidate {
|
||||
pub instance: String,
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub version: Option<String>,
|
||||
pub resource_id: String,
|
||||
pub caps: Vec<String>,
|
||||
/// `endpoints.http` — what the consumer would register.
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub binds: Option<String>,
|
||||
pub state: CandidateState,
|
||||
/// The list entry that already holds this URL (state `configured`).
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub configured_as: Option<String>,
|
||||
/// A list-safe name to register it under.
|
||||
pub suggested_name: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScanSummary {
|
||||
/// Every node heard on the segment, whatever it advertises.
|
||||
pub nodes: usize,
|
||||
/// Resources matching the consumer's pattern.
|
||||
pub capable: usize,
|
||||
pub configured: usize,
|
||||
pub bound_elsewhere: usize,
|
||||
pub addable: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ScanReport {
|
||||
pub cap: String,
|
||||
/// This consumer's instance (what `binds` is compared against).
|
||||
#[serde(rename = "self")]
|
||||
pub self_instance: String,
|
||||
pub duration_ms: u64,
|
||||
pub summary: ScanSummary,
|
||||
pub candidates: Vec<Candidate>,
|
||||
}
|
||||
|
||||
fn norm(u: &str) -> String {
|
||||
u.trim().trim_end_matches('/').to_string()
|
||||
}
|
||||
|
||||
/// Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules.
|
||||
pub fn suggested_name(instance: &str) -> String {
|
||||
let s: String = instance
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '-' })
|
||||
.collect();
|
||||
if s.is_empty() { "plugin".into() } else { s }
|
||||
}
|
||||
|
||||
/// Pure classification, for tests and for callers that already hold a view.
|
||||
/// `configured` is `(name, url)` of every entry the consumer has.
|
||||
pub fn classify(
|
||||
pattern: &str,
|
||||
self_instance: &str,
|
||||
nodes: usize,
|
||||
resources: Vec<SeenResource>,
|
||||
configured: &[(String, String)],
|
||||
) -> ScanReport {
|
||||
let mut summary = ScanSummary { nodes, ..Default::default() };
|
||||
let mut candidates = Vec::new();
|
||||
for r in resources {
|
||||
// A resource without an http endpoint cannot be registered anywhere.
|
||||
let Some(url) = r.resource.endpoint("http") else { continue };
|
||||
let url = norm(&url);
|
||||
summary.capable += 1;
|
||||
let configured_as = configured.iter().find(|(_, u)| norm(u) == url).map(|(n, _)| n.clone());
|
||||
let state = if configured_as.is_some() {
|
||||
summary.configured += 1;
|
||||
CandidateState::Configured
|
||||
} else if !r.resource.usable_by(self_instance) {
|
||||
summary.bound_elsewhere += 1;
|
||||
CandidateState::BoundElsewhere
|
||||
} else {
|
||||
summary.addable += 1;
|
||||
CandidateState::Addable
|
||||
};
|
||||
candidates.push(Candidate {
|
||||
suggested_name: suggested_name(&r.node.instance),
|
||||
instance: r.node.instance,
|
||||
name: r.node.name,
|
||||
version: r.node.version,
|
||||
resource_id: r.resource.id,
|
||||
caps: r.resource.caps,
|
||||
url,
|
||||
binds: r.resource.binds,
|
||||
state,
|
||||
configured_as,
|
||||
});
|
||||
}
|
||||
// Addable first (what the operator came for), then configured, then the rest.
|
||||
let rank = |s: CandidateState| match s {
|
||||
CandidateState::Addable => 0,
|
||||
CandidateState::Configured => 1,
|
||||
CandidateState::BoundElsewhere => 2,
|
||||
};
|
||||
candidates.sort_by(|a, b| rank(a.state).cmp(&rank(b.state)).then(a.instance.cmp(&b.instance)));
|
||||
ScanReport {
|
||||
cap: pattern.to_string(),
|
||||
self_instance: self_instance.to_string(),
|
||||
duration_ms: 0,
|
||||
summary,
|
||||
candidates,
|
||||
}
|
||||
}
|
||||
|
||||
/// Probe every node, wait `wait` for the replies, then classify what matches
|
||||
/// `pattern`. A node that stays silent through the wait but is still inside
|
||||
/// its liveness window is counted too — the view is "who is here", not "who
|
||||
/// answered this probe".
|
||||
pub async fn scan(
|
||||
node: &BeaconNode,
|
||||
pattern: &str,
|
||||
configured: &[(String, String)],
|
||||
wait: Duration,
|
||||
) -> ScanReport {
|
||||
let started = Instant::now();
|
||||
node.probe(&[]).await;
|
||||
tokio::time::sleep(wait).await;
|
||||
let mut report = classify(
|
||||
pattern,
|
||||
&node.instance(),
|
||||
node.nodes().len(),
|
||||
node.resources(pattern),
|
||||
configured,
|
||||
);
|
||||
report.duration_ms = started.elapsed().as_millis() as u64;
|
||||
report
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::beacon::{NodeInfo, Resource};
|
||||
|
||||
fn seen(instance: &str, cap: &str, url: Option<&str>, binds: Option<&str>) -> SeenResource {
|
||||
let mut r = Resource::new("x", [cap]);
|
||||
if let Some(u) = url {
|
||||
r = r.with_endpoint("http", u);
|
||||
}
|
||||
r.binds = binds.map(Into::into);
|
||||
SeenResource {
|
||||
node: NodeInfo { name: "plugin".into(), instance: instance.into(), ..Default::default() },
|
||||
resource: r,
|
||||
addr: "10.0.0.1".into(),
|
||||
last_seen: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_configured_addable_and_bound_elsewhere() {
|
||||
let resources = vec![
|
||||
seen("metashare-nzb", "metamesh.transport/nzb@1", Some("http://metashare-nzb:3000/"), Some("metashare-app")),
|
||||
seen("qbit", "metamesh.transport/torrent@1", Some("http://qbit:3000"), None),
|
||||
seen("other-ipfs", "metamesh.transport/ipfs@1", Some("http://other-ipfs:3000"), Some("metawatch-share")),
|
||||
seen("no-endpoint", "metamesh.transport/ipfs@1", None, None),
|
||||
];
|
||||
let configured = vec![("nzb".to_string(), "http://metashare-nzb:3000".to_string())];
|
||||
let r = classify("metamesh.transport/*", "metashare-app", 10, resources, &configured);
|
||||
assert_eq!(r.summary.nodes, 10);
|
||||
assert_eq!(r.summary.capable, 3, "a resource without an http endpoint is not a candidate");
|
||||
assert_eq!((r.summary.configured, r.summary.addable, r.summary.bound_elsewhere), (1, 1, 1));
|
||||
assert_eq!(r.candidates[0].instance, "qbit", "addable rows sort first");
|
||||
assert_eq!(r.candidates[1].configured_as.as_deref(), Some("nzb"));
|
||||
assert_eq!(r.candidates[2].state, CandidateState::BoundElsewhere);
|
||||
let json = serde_json::to_value(&r).unwrap();
|
||||
assert_eq!(json["self"], "metashare-app");
|
||||
assert_eq!(json["summary"]["boundElsewhere"], 1);
|
||||
assert_eq!(json["candidates"][2]["state"], "bound-elsewhere");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggested_names_are_list_safe() {
|
||||
assert_eq!(suggested_name("MetaShare NZB"), "metashare-nzb");
|
||||
assert_eq!(suggested_name(""), "plugin");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_reports_what_the_node_heard() {
|
||||
use crate::beacon::{BeaconNode, Config, Message};
|
||||
let port = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
||||
let mut cfg = Config::new("consumer").instance("me");
|
||||
cfg.port = port;
|
||||
let node = BeaconNode::spawn(cfg).unwrap();
|
||||
let ad = Message::advertise(
|
||||
NodeInfo { name: "meta-feeder-tmdb".into(), instance: "tmdb-feeder".into(), ..Default::default() },
|
||||
vec![Resource::new("feeder", ["metamesh.feeder/tmdb"]).with_endpoint("http", "http://tmdb-feeder:8080")],
|
||||
);
|
||||
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
|
||||
s.send_to(&serde_json::to_vec(&ad).unwrap(), ("127.0.0.1", port)).await.unwrap();
|
||||
let r = scan(&node, "metamesh.feeder/*", &[], Duration::from_millis(200)).await;
|
||||
assert_eq!(r.summary.nodes, 1);
|
||||
assert_eq!(r.summary.addable, 1);
|
||||
assert_eq!(r.candidates[0].url, "http://tmdb-feeder:8080");
|
||||
assert_eq!(r.candidates[0].suggested_name, "tmdb-feeder");
|
||||
assert!(r.duration_ms >= 200);
|
||||
}
|
||||
}
|
||||
+602
@@ -0,0 +1,602 @@
|
||||
//! MMUR2 signed claims: signer `uid` says `key = value` about content `cid`, at
|
||||
//! `ts` (milliseconds, also the version). See meta-root
|
||||
//! `docs/implementation/metacore-claims-model.md` §3.
|
||||
//!
|
||||
//! Wire spec (pinned by `tests/claim-vectors.json`, vendored from meta-core's
|
||||
//! `internal/claim/testdata` — `scripts/check-claim-vectors.sh` guards it):
|
||||
//!
|
||||
//! ```text
|
||||
//! domain = b"MMUR2\n"
|
||||
//! payload = canonical CBOR array [cid, uid, key, value, ts]
|
||||
//! bytes_to_sign = domain || payload
|
||||
//! sig = ECDSA-secp256k1(SHA-256(bytes_to_sign)), compact r||s, low-S
|
||||
//! ```
|
||||
//!
|
||||
//! `value` is CBOR text (public), bytes (private envelope) or null (retraction).
|
||||
//! The encoder is hand-rolled: those five shapes are all MMUR2 needs, and a
|
||||
//! hand-rolled head encoder cannot drift from "smallest integer form".
|
||||
//!
|
||||
//! The type and its JSON form are always available with the `feeder` feature
|
||||
//! (the gateway wire carries claims); signing and verification are here too,
|
||||
//! behind `claims`, which `feeder` enables.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use base64::Engine as _;
|
||||
use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
|
||||
|
||||
/// Domain separator. Changing it invalidates every claim everywhere.
|
||||
pub const DOMAIN: &[u8] = b"MMUR2\n";
|
||||
|
||||
/// A claim's value. The SQLite storage type in meta-core mirrors it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||
pub enum ClaimValue {
|
||||
/// Retraction (tombstone).
|
||||
#[default]
|
||||
Null,
|
||||
/// Public plaintext.
|
||||
Text(String),
|
||||
/// Private envelope (XChaCha20-Poly1305), opaque to everyone but the owner.
|
||||
Bytes(Vec<u8>),
|
||||
}
|
||||
|
||||
impl ClaimValue {
|
||||
pub fn is_null(&self) -> bool {
|
||||
matches!(self, ClaimValue::Null)
|
||||
}
|
||||
pub fn as_text(&self) -> Option<&str> {
|
||||
match self {
|
||||
ClaimValue::Text(s) => Some(s),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One signed vote.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SignedClaim {
|
||||
pub cid: String,
|
||||
pub uid: String,
|
||||
pub key: String,
|
||||
#[serde(with = "value_json")]
|
||||
pub value: ClaimValue,
|
||||
pub ts: u64,
|
||||
/// 64-byte compact r||s, base64 in JSON.
|
||||
#[serde(with = "sig_b64")]
|
||||
pub sig: Vec<u8>,
|
||||
}
|
||||
|
||||
impl SignedClaim {
|
||||
/// The exact bytes the signature covers.
|
||||
pub fn signed_bytes(&self) -> Vec<u8> {
|
||||
signed_bytes(&self.cid, &self.uid, &self.key, &self.value, self.ts)
|
||||
}
|
||||
}
|
||||
|
||||
/// `domain || canonical CBOR [cid, uid, key, value, ts]`.
|
||||
pub fn signed_bytes(cid: &str, uid: &str, key: &str, value: &ClaimValue, ts: u64) -> Vec<u8> {
|
||||
let extra = match value {
|
||||
ClaimValue::Text(s) => s.len(),
|
||||
ClaimValue::Bytes(b) => b.len(),
|
||||
ClaimValue::Null => 0,
|
||||
};
|
||||
let mut b = Vec::with_capacity(DOMAIN.len() + cid.len() + uid.len() + key.len() + extra + 32);
|
||||
b.extend_from_slice(DOMAIN);
|
||||
head(&mut b, 4, 5);
|
||||
text(&mut b, cid);
|
||||
text(&mut b, uid);
|
||||
text(&mut b, key);
|
||||
match value {
|
||||
ClaimValue::Text(s) => text(&mut b, s),
|
||||
ClaimValue::Bytes(bytes) => {
|
||||
head(&mut b, 2, bytes.len() as u64);
|
||||
b.extend_from_slice(bytes);
|
||||
}
|
||||
ClaimValue::Null => b.push(0xf6),
|
||||
}
|
||||
head(&mut b, 0, ts);
|
||||
b
|
||||
}
|
||||
|
||||
fn text(b: &mut Vec<u8>, s: &str) {
|
||||
head(b, 3, s.len() as u64);
|
||||
b.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
/// CBOR major type + argument, smallest form.
|
||||
fn head(b: &mut Vec<u8>, major: u8, n: u64) {
|
||||
let m = major << 5;
|
||||
if n < 24 {
|
||||
b.push(m | n as u8);
|
||||
} else if n <= 0xff {
|
||||
b.extend_from_slice(&[m | 24, n as u8]);
|
||||
} else if n <= 0xffff {
|
||||
b.push(m | 25);
|
||||
b.extend_from_slice(&(n as u16).to_be_bytes());
|
||||
} else if n <= 0xffff_ffff {
|
||||
b.push(m | 26);
|
||||
b.extend_from_slice(&(n as u32).to_be_bytes());
|
||||
} else {
|
||||
b.push(m | 27);
|
||||
b.extend_from_slice(&n.to_be_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON (human-readable) uses the spec §3.4 form — a string, `{"b64": …}` or
|
||||
/// `null`. Binary formats (the bincode gateway wire) use a plain tagged enum:
|
||||
/// bincode cannot decode the untagged JSON shape.
|
||||
mod value_json {
|
||||
use super::*;
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct B64 {
|
||||
b64: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
enum Wire {
|
||||
Null,
|
||||
Text(String),
|
||||
Bytes(Vec<u8>),
|
||||
}
|
||||
|
||||
pub fn serialize<S: Serializer>(v: &ClaimValue, s: S) -> Result<S::Ok, S::Error> {
|
||||
if !s.is_human_readable() {
|
||||
let w = match v {
|
||||
ClaimValue::Null => Wire::Null,
|
||||
ClaimValue::Text(t) => Wire::Text(t.clone()),
|
||||
ClaimValue::Bytes(b) => Wire::Bytes(b.clone()),
|
||||
};
|
||||
return w.serialize(s);
|
||||
}
|
||||
match v {
|
||||
ClaimValue::Null => s.serialize_none(),
|
||||
ClaimValue::Text(t) => s.serialize_str(t),
|
||||
ClaimValue::Bytes(b) => B64 {
|
||||
b64: base64::engine::general_purpose::STANDARD.encode(b),
|
||||
}
|
||||
.serialize(s),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<ClaimValue, D::Error> {
|
||||
if !d.is_human_readable() {
|
||||
return Ok(match Wire::deserialize(d)? {
|
||||
Wire::Null => ClaimValue::Null,
|
||||
Wire::Text(t) => ClaimValue::Text(t),
|
||||
Wire::Bytes(b) => ClaimValue::Bytes(b),
|
||||
});
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Raw {
|
||||
Text(String),
|
||||
Bytes(B64),
|
||||
}
|
||||
Ok(match Option::<Raw>::deserialize(d)? {
|
||||
None => ClaimValue::Null,
|
||||
Some(Raw::Text(t)) => ClaimValue::Text(t),
|
||||
Some(Raw::Bytes(B64 { b64 })) => ClaimValue::Bytes(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(b64)
|
||||
.map_err(de::Error::custom)?,
|
||||
),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// base64 in JSON, raw bytes in binary formats.
|
||||
mod sig_b64 {
|
||||
use super::*;
|
||||
|
||||
pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
|
||||
if !s.is_human_readable() {
|
||||
// A plain sequence, symmetric with `Vec::<u8>::deserialize` below.
|
||||
return v.serialize(s);
|
||||
}
|
||||
s.serialize_str(&base64::engine::general_purpose::STANDARD.encode(v))
|
||||
}
|
||||
|
||||
pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
|
||||
if !d.is_human_readable() {
|
||||
return Vec::<u8>::deserialize(d);
|
||||
}
|
||||
let s = String::deserialize(d)?;
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(s)
|
||||
.map_err(de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
/// Union-flatten a claim list into a field map: every key that has at least
|
||||
/// one non-retracted text vote, with the first such value seen. A
|
||||
/// **superset view for filtering and scoring**, not a merge — the merge (most
|
||||
/// signers → oldest ts → smallest value, own vote first) lives only in
|
||||
/// meta-core (`POST /api/merge`). Ciphertext never appears.
|
||||
pub fn union_fields(claims: &[SignedClaim]) -> BTreeMap<String, String> {
|
||||
let mut out = BTreeMap::new();
|
||||
for c in claims {
|
||||
if let ClaimValue::Text(v) = &c.value {
|
||||
out.entry(c.key.clone()).or_insert_with(|| v.clone());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(feature = "claims")]
|
||||
pub use crypto::*;
|
||||
|
||||
#[cfg(feature = "claims")]
|
||||
mod crypto {
|
||||
use super::*;
|
||||
|
||||
use k256::ecdsa::signature::hazmat::{PrehashSigner, PrehashVerifier};
|
||||
use k256::ecdsa::{Signature, SigningKey, VerifyingKey};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// Why a claim failed verification.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ClaimError {
|
||||
#[error("claim: missing cid or key")]
|
||||
Shape,
|
||||
#[error("uid: {0}")]
|
||||
Uid(String),
|
||||
#[error("sig: want a 64-byte compact signature")]
|
||||
SigLen,
|
||||
#[error("sig: high-S signatures are refused (MMUR2 is low-S only)")]
|
||||
HighS,
|
||||
#[error("sig: does not verify")]
|
||||
Invalid,
|
||||
}
|
||||
|
||||
/// Recover the verifying key a uid encodes (`"z" + base58btc(33-byte sec1)`).
|
||||
pub fn pubkey_from_uid(uid: &str) -> Result<VerifyingKey, ClaimError> {
|
||||
let body = uid
|
||||
.strip_prefix('z')
|
||||
.ok_or_else(|| ClaimError::Uid("missing multibase 'z' prefix".into()))?;
|
||||
let raw = bs58::decode(body)
|
||||
.into_vec()
|
||||
.map_err(|e| ClaimError::Uid(e.to_string()))?;
|
||||
if raw.len() != 33 {
|
||||
return Err(ClaimError::Uid(format!(
|
||||
"pubkey is {} bytes, want 33",
|
||||
raw.len()
|
||||
)));
|
||||
}
|
||||
VerifyingKey::from_sec1_bytes(&raw).map_err(|e| ClaimError::Uid(e.to_string()))
|
||||
}
|
||||
|
||||
/// The uid of a verifying key.
|
||||
pub fn uid_from_pubkey(vk: &VerifyingKey) -> String {
|
||||
let point = vk.to_encoded_point(true);
|
||||
format!("z{}", bs58::encode(point.as_bytes()).into_string())
|
||||
}
|
||||
|
||||
impl SignedClaim {
|
||||
/// Check shape, low-S and the signature against the uid's key.
|
||||
pub fn verify(&self) -> Result<(), ClaimError> {
|
||||
if self.cid.is_empty() || self.key.is_empty() {
|
||||
return Err(ClaimError::Shape);
|
||||
}
|
||||
if self.sig.len() != 64 {
|
||||
return Err(ClaimError::SigLen);
|
||||
}
|
||||
let sig = Signature::from_slice(&self.sig).map_err(|_| ClaimError::Invalid)?;
|
||||
if sig.normalize_s().is_some() {
|
||||
return Err(ClaimError::HighS);
|
||||
}
|
||||
let vk = pubkey_from_uid(&self.uid)?;
|
||||
let digest = Sha256::digest(self.signed_bytes());
|
||||
vk.verify_prehash(&digest, &sig)
|
||||
.map_err(|_| ClaimError::Invalid)
|
||||
}
|
||||
}
|
||||
|
||||
/// A writer's identity: one key per writing container (spec D2).
|
||||
#[derive(Clone)]
|
||||
pub struct Signer {
|
||||
key: SigningKey,
|
||||
uid: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Signer {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Signer")
|
||||
.field("uid", &self.uid)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Signer {
|
||||
/// From a 32-byte secret, hex (optionally `0x`-prefixed).
|
||||
pub fn from_hex(hex_key: &str) -> anyhow::Result<Self> {
|
||||
let s = hex_key.trim();
|
||||
let s = s
|
||||
.strip_prefix("0x")
|
||||
.or_else(|| s.strip_prefix("0X"))
|
||||
.unwrap_or(s);
|
||||
let raw = decode_hex(s).ok_or_else(|| anyhow::anyhow!("signing key is not hex"))?;
|
||||
if raw.len() != 32 {
|
||||
anyhow::bail!("signing key must be 32 bytes, got {}", raw.len());
|
||||
}
|
||||
let key = SigningKey::from_slice(&raw)
|
||||
.map_err(|e| anyhow::anyhow!("invalid signing key: {e}"))?;
|
||||
let uid = uid_from_pubkey(key.verifying_key());
|
||||
Ok(Signer { key, uid })
|
||||
}
|
||||
|
||||
/// A fresh random key.
|
||||
pub fn generate() -> Self {
|
||||
let key = SigningKey::random(&mut rand_core::OsRng);
|
||||
let uid = uid_from_pubkey(key.verifying_key());
|
||||
Signer { key, uid }
|
||||
}
|
||||
|
||||
pub fn uid(&self) -> &str {
|
||||
&self.uid
|
||||
}
|
||||
|
||||
/// The secret as lowercase hex (for writing a generated key file).
|
||||
pub fn secret_hex(&self) -> String {
|
||||
self.key
|
||||
.to_bytes()
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Sign one claim at `ts`. RFC 6979 deterministic, normalised to low-S.
|
||||
pub fn sign(&self, cid: &str, key: &str, value: ClaimValue, ts: u64) -> SignedClaim {
|
||||
let bytes = signed_bytes(cid, &self.uid, key, &value, ts);
|
||||
let digest = Sha256::digest(&bytes);
|
||||
let sig: Signature = self
|
||||
.key
|
||||
.sign_prehash(&digest)
|
||||
.expect("a 32-byte prehash always signs");
|
||||
let sig = sig.normalize_s().unwrap_or(sig);
|
||||
SignedClaim {
|
||||
cid: cid.to_string(),
|
||||
uid: self.uid.clone(),
|
||||
key: key.to_string(),
|
||||
value,
|
||||
ts,
|
||||
sig: sig.to_bytes().to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sign every field of a flat map as text claims on `cid`, all at `ts`.
|
||||
pub fn sign_fields(
|
||||
&self,
|
||||
cid: &str,
|
||||
fields: &BTreeMap<String, String>,
|
||||
ts: u64,
|
||||
) -> Vec<SignedClaim> {
|
||||
fields
|
||||
.iter()
|
||||
.map(|(k, v)| self.sign(cid, k, ClaimValue::Text(v.clone()), ts))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A signed retraction of `key` on `cid`.
|
||||
pub fn retract(&self, cid: &str, key: &str, ts: u64) -> SignedClaim {
|
||||
self.sign(cid, key, ClaimValue::Null, ts)
|
||||
}
|
||||
}
|
||||
|
||||
/// Milliseconds since the epoch — the `ts` of a claim signed now. Strictly
|
||||
/// increasing per process: `ts` is the version, so two writes to one key in
|
||||
/// the same millisecond must not make the second one `stale`.
|
||||
pub fn now_ms() -> u64 {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static LAST: AtomicU64 = AtomicU64::new(0);
|
||||
let wall = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let mut last = LAST.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let next = wall.max(last + 1);
|
||||
match LAST.compare_exchange_weak(last, next, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => return next,
|
||||
Err(seen) => last = seen,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Env var holding the signer's secret as hex.
|
||||
pub const SIGNING_KEY_ENV: &str = "METAMESH_SIGNING_KEY";
|
||||
/// Env var naming a key file (bind-mounted); generated on first boot.
|
||||
pub const SIGNING_KEY_FILE_ENV: &str = "METAMESH_SIGNING_KEY_FILE";
|
||||
|
||||
/// The writer's key (spec §9): `METAMESH_SIGNING_KEY` (hex) wins; else
|
||||
/// `METAMESH_SIGNING_KEY_FILE`, generated and written (mode 0600) when the
|
||||
/// file does not exist yet. Neither set → error: a writer must never sign
|
||||
/// with a silent ephemeral identity.
|
||||
pub fn load_or_create_key() -> anyhow::Result<Signer> {
|
||||
if let Some(hex) = env_nonempty(SIGNING_KEY_ENV) {
|
||||
return Signer::from_hex(&hex);
|
||||
}
|
||||
let Some(path) = env_nonempty(SIGNING_KEY_FILE_ENV) else {
|
||||
anyhow::bail!("no signing key: set {SIGNING_KEY_ENV} or {SIGNING_KEY_FILE_ENV}");
|
||||
};
|
||||
load_or_create_key_file(std::path::Path::new(&path))
|
||||
}
|
||||
|
||||
/// Read a hex key file, or generate one there (mode 0600) if absent.
|
||||
pub fn load_or_create_key_file(path: &std::path::Path) -> anyhow::Result<Signer> {
|
||||
match std::fs::read_to_string(path) {
|
||||
Ok(s) => Signer::from_hex(&s),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
let signer = Signer::generate();
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
}
|
||||
write_secret(path, &signer.secret_hex())?;
|
||||
tracing::info!(target: "meta-feeder::claims", uid = %signer.uid(),
|
||||
path = %path.display(), "generated a new signing key");
|
||||
Ok(signer)
|
||||
}
|
||||
Err(e) => Err(anyhow::anyhow!("read signing key {}: {e}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
|
||||
use std::io::Write;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
let mut f = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)?;
|
||||
f.write_all(hex.as_bytes())?;
|
||||
f.write_all(b"\n")
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn write_secret(path: &std::path::Path, hex: &str) -> std::io::Result<()> {
|
||||
std::fs::write(path, format!("{hex}\n"))
|
||||
}
|
||||
|
||||
fn env_nonempty(key: &str) -> Option<String> {
|
||||
std::env::var(key)
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
fn decode_hex(s: &str) -> Option<Vec<u8>> {
|
||||
if s.len() % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
(0..s.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Vector {
|
||||
name: String,
|
||||
cid: String,
|
||||
uid: String,
|
||||
key: String,
|
||||
value: Option<String>,
|
||||
#[serde(rename = "valueBytesHex")]
|
||||
value_bytes_hex: Option<String>,
|
||||
ts: u64,
|
||||
#[serde(rename = "signedBytesHex")]
|
||||
signed: String,
|
||||
#[serde(rename = "sigHex")]
|
||||
sig: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct HighS {
|
||||
vector: String,
|
||||
#[serde(rename = "sigHex")]
|
||||
sig: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct File {
|
||||
#[serde(rename = "privateKeyHex")]
|
||||
private_key_hex: String,
|
||||
uid: String,
|
||||
vectors: Vec<Vector>,
|
||||
#[serde(rename = "highS")]
|
||||
high_s: HighS,
|
||||
}
|
||||
|
||||
fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
fn load() -> File {
|
||||
serde_json::from_str(include_str!("../tests/claim-vectors.json")).unwrap()
|
||||
}
|
||||
|
||||
fn value(v: &Vector) -> ClaimValue {
|
||||
match (&v.value_bytes_hex, &v.value) {
|
||||
(Some(h), _) => ClaimValue::Bytes(decode_hex(h).unwrap()),
|
||||
(None, Some(t)) => ClaimValue::Text(t.clone()),
|
||||
(None, None) => ClaimValue::Null,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn golden_vectors() {
|
||||
let f = load();
|
||||
let signer = Signer::from_hex(&f.private_key_hex).unwrap();
|
||||
assert_eq!(signer.uid(), f.uid);
|
||||
for v in &f.vectors {
|
||||
let c = signer.sign(&v.cid, &v.key, value(v), v.ts);
|
||||
assert_eq!(c.uid, v.uid, "{}", v.name);
|
||||
assert_eq!(hex(&c.signed_bytes()), v.signed, "{}: signed bytes", v.name);
|
||||
assert_eq!(hex(&c.sig), v.sig, "{}: RFC 6979 signature", v.name);
|
||||
c.verify().unwrap_or_else(|e| panic!("{}: {e}", v.name));
|
||||
// JSON round trip keeps the claim verifiable.
|
||||
let back: SignedClaim =
|
||||
serde_json::from_str(&serde_json::to_string(&c).unwrap()).unwrap();
|
||||
assert_eq!(back, c, "{}: json", v.name);
|
||||
}
|
||||
let v = f
|
||||
.vectors
|
||||
.iter()
|
||||
.find(|v| v.name == f.high_s.vector)
|
||||
.unwrap();
|
||||
let high = SignedClaim {
|
||||
cid: v.cid.clone(),
|
||||
uid: v.uid.clone(),
|
||||
key: v.key.clone(),
|
||||
value: value(v),
|
||||
ts: v.ts,
|
||||
sig: decode_hex(&f.high_s.sig).unwrap(),
|
||||
};
|
||||
assert!(matches!(high.verify(), Err(ClaimError::HighS)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tamper_is_refused() {
|
||||
let s = Signer::generate();
|
||||
let mut c = s.sign("bafy", "title", ClaimValue::Text("A".into()), 1);
|
||||
c.value = ClaimValue::Text("B".into());
|
||||
assert!(matches!(c.verify(), Err(ClaimError::Invalid)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_file_is_generated_once() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let p = dir.path().join("keys/signing.key");
|
||||
let a = load_or_create_key_file(&p).unwrap();
|
||||
let b = load_or_create_key_file(&p).unwrap();
|
||||
assert_eq!(a.uid(), b.uid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_shapes() {
|
||||
let s = Signer::generate();
|
||||
let j =
|
||||
serde_json::to_value(s.sign("c", "k", ClaimValue::Bytes(vec![1, 2]), 5)).unwrap();
|
||||
assert_eq!(j["value"]["b64"], "AQI=");
|
||||
let j = serde_json::to_value(s.retract("c", "k", 6)).unwrap();
|
||||
assert!(j["value"].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn union_keeps_first_text_and_skips_retractions_and_cipher() {
|
||||
let s = Signer::generate();
|
||||
let claims = vec![
|
||||
s.sign("c", "title", ClaimValue::Text("A".into()), 1),
|
||||
s.sign("c", "title", ClaimValue::Text("B".into()), 2),
|
||||
s.retract("c", "gone", 3),
|
||||
s.sign("c", "seek", ClaimValue::Bytes(vec![9]), 4),
|
||||
];
|
||||
let f = union_fields(&claims);
|
||||
assert_eq!(f.get("title").map(String::as_str), Some("A"));
|
||||
assert!(!f.contains_key("gone") && !f.contains_key("seek"));
|
||||
}
|
||||
}
|
||||
}
|
||||
+31
-7
@@ -64,6 +64,10 @@ pub struct EnrichmentConfig {
|
||||
pub opensubtitles_password: Option<String>,
|
||||
/// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`).
|
||||
pub opensubtitles_languages: String,
|
||||
/// This feeder's claim signer (`METAMESH_SIGNING_KEY` / `_FILE`). `None`
|
||||
/// when no key is configured: every meta-core write then fails loudly
|
||||
/// instead of publishing unsigned data.
|
||||
pub signer: Option<crate::claims::Signer>,
|
||||
}
|
||||
|
||||
fn env_nonempty(key: &str) -> Option<String> {
|
||||
@@ -120,6 +124,14 @@ impl EnrichmentConfig {
|
||||
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
|
||||
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
|
||||
.unwrap_or_else(|| "en".to_string()),
|
||||
signer: match crate::claims::load_or_create_key() {
|
||||
Ok(s) => Some(s),
|
||||
Err(e) => {
|
||||
warn!(target: "meta-feeder::enrich", error = %e,
|
||||
"no signing key; meta-core writes will fail");
|
||||
None
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -218,7 +230,14 @@ impl Enricher {
|
||||
&self.cfg.gateway_peer,
|
||||
);
|
||||
let body = meta_core::build_metadata_body(record, &prov);
|
||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await
|
||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, self.signer()?, cid, &body).await
|
||||
}
|
||||
|
||||
fn signer(&self) -> anyhow::Result<&crate::claims::Signer> {
|
||||
self.cfg
|
||||
.signer
|
||||
.as_ref()
|
||||
.ok_or_else(|| anyhow::anyhow!("no signing key (METAMESH_SIGNING_KEY / _FILE)"))
|
||||
}
|
||||
|
||||
/// Run the video enrichment pipeline for one resolved CID: filename-parser
|
||||
@@ -240,14 +259,14 @@ impl Enricher {
|
||||
Ok(()) => {
|
||||
// Poll for the merged fields so tmdb gets the clean title.
|
||||
if let Some(merged) = self
|
||||
.poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||
.poll_until_field(cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||
.await
|
||||
{
|
||||
meta = merged;
|
||||
} else {
|
||||
debug!(
|
||||
target: "meta-feeder::enrich",
|
||||
cid, "filename-parser produced no originalTitle within timeout; \
|
||||
cid, "filename-parser produced no parsedTitle within timeout; \
|
||||
handing tmdb the base fields"
|
||||
);
|
||||
}
|
||||
@@ -324,7 +343,7 @@ impl Enricher {
|
||||
|
||||
// Poll the primary for filename-parser's clean title, then tmdb it.
|
||||
let merged = self
|
||||
.poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||
.poll_until_field(&p.cid, "parsedTitle", FILENAME_PARSE_POLL_TIMEOUT)
|
||||
.await
|
||||
.unwrap_or_else(|| p.base.clone());
|
||||
|
||||
@@ -355,9 +374,14 @@ impl Enricher {
|
||||
}
|
||||
// PUT is merge/upsert — this layers the show-level anchor onto each
|
||||
// file's record without disturbing its per-file fields.
|
||||
if let Err(e) =
|
||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await
|
||||
{
|
||||
let res = match self.signer() {
|
||||
Ok(signer) => {
|
||||
meta_core::put_record(&self.http, &self.cfg.meta_core_url, signer, &v.cid, &anchor)
|
||||
.await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
if let Err(e) = res {
|
||||
warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -728,6 +728,159 @@ pub fn extract_codec(title: &str) -> Option<String> {
|
||||
Some(norm.to_string())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Release facts — `sourceMedium`, `releaseGroup`, `editions/*`, `flags/*`
|
||||
// (METADATA_KEYS §3). Durable properties of a release (unlike its seeders), so
|
||||
// they belong on the record; title-derived and best-effort like `quality` and
|
||||
// `codec`, so absence means "couldn't tell". Each returns a closed vocabulary
|
||||
// and drops what it does not recognise. The same functions normalise a
|
||||
// structured value an upstream already parsed (AIOStreams' `quality`
|
||||
// "BluRay REMUX", its editions "Director's Cut"), so the two paths agree.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Separators to spaces, the way every extractor here reads a title. Hyphens
|
||||
/// are kept: `WEB-DL`, `HD-Rip` and the `-GROUP` suffix need them.
|
||||
fn spaced(title: &str) -> String {
|
||||
title.replace(['.', '_'], " ")
|
||||
}
|
||||
|
||||
/// The medium the release was taken from, one of `cam`, `ts`, `tc`, `scr`,
|
||||
/// `bluray-remux`, `bluray`, `webrip`, `web-dl`, `hdrip`, `hdtv`, `dvd`.
|
||||
///
|
||||
/// Pre-release sources are tested first: `HDCAM` beats any `HD` token, and a
|
||||
/// `DVDSCR` is a screener, not a DVD. The bare `CAM`, `TS` and `TC` tokens only
|
||||
/// count in capitals: as words they are a film title (*Cam*, 2018), a file
|
||||
/// extension and an initialism too often.
|
||||
pub fn extract_source_medium(title: &str) -> Option<&'static str> {
|
||||
static CI: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
[
|
||||
(r"(?i)\b(?:hd-?cam|cam-?rip)\b", "cam"),
|
||||
(r"(?i)\b(?:telesync|hd-?ts|pdvd)\b", "ts"),
|
||||
(r"(?i)\b(?:telecine|hd-?tc)\b", "tc"),
|
||||
(r"(?i)\b(?:dvd-?scr|screener|scr)\b", "scr"),
|
||||
(r"(?i)\b(?:bd-?remux|remux)\b", "bluray-remux"),
|
||||
(r"(?i)\b(?:blu-?ray|bd-?rip|br-?rip|bdmv|bd25|bd50)\b", "bluray"),
|
||||
(r"(?i)\bweb-?rip\b", "webrip"),
|
||||
(r"(?i)\b(?:web-?dl|web)\b", "web-dl"),
|
||||
(r"(?i)\bhd-?rip\b", "hdrip"),
|
||||
(r"(?i)\b(?:hdtv|hdtvrip|uhdtv|pdtv|sdtv|tvrip|dsr)\b", "hdtv"),
|
||||
(r"(?i)\b(?:dvd-?rip|dvd-?r|dvd5|dvd9|dvd)\b", "dvd"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(re, v)| (Regex::new(re).expect("source re compiles"), v))
|
||||
.collect()
|
||||
});
|
||||
static CAPS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
[(r"\bCAM\b", "cam"), (r"\bTS\b", "ts"), (r"\bTC\b", "tc")]
|
||||
.into_iter()
|
||||
.map(|(re, v)| (Regex::new(re).expect("caps re compiles"), v))
|
||||
.collect()
|
||||
});
|
||||
static DVD_REMUX: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"(?i)\bdvd\S*\b.*\bremux\b|\bremux\b.*\bdvd").expect("dvd remux re"));
|
||||
let t = spaced(title);
|
||||
// A DVD remux is a DVD: `remux` alone names no disc.
|
||||
if DVD_REMUX.is_match(&t) {
|
||||
return Some("dvd");
|
||||
}
|
||||
CAPS.iter()
|
||||
.find(|(re, _)| re.is_match(&t))
|
||||
.or_else(|| CI.iter().find(|(re, _)| re.is_match(&t)))
|
||||
.map(|(_, v)| *v)
|
||||
}
|
||||
|
||||
/// The group that made the release: a leading `[Group]` (fansub style) or a
|
||||
/// trailing `-GROUP` (scene style), the same two rules as meta-watch's
|
||||
/// `parseVersion`. Trailing tracker tags (`[rarbg]`, `[TGx]`), a video
|
||||
/// extension and a usenet `-xpost` are removed first; a candidate that is
|
||||
/// really a format token (`WEB-DL` → `DL`, `x264`, `1080p`) is refused.
|
||||
pub fn extract_release_group(title: &str) -> Option<String> {
|
||||
static LEADING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\s*\[([^\[\]]{2,32})\]").expect("leading group re"));
|
||||
static TRAILING: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"-\s*([A-Za-z0-9][A-Za-z0-9&@]{1,24})\s*$").expect("trailing group re"));
|
||||
static TAIL_NOISE: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?i)(?:\s*(?:\[[^\]]*\]|\([^)]*\))|\.(?:mkv|mp4|avi|m4v|ts|m2ts|webm|wmv|mov)|-xpost)+\s*$")
|
||||
.expect("tail noise re")
|
||||
});
|
||||
static FORMAT: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^(?:[xh]26[45]|hevc|avc|av1|\d{3,4}[pi]|s\d+(?:e\d+)?)$").expect("format re")
|
||||
});
|
||||
let not_a_format = |g: &str| {
|
||||
let l = g.to_ascii_lowercase();
|
||||
!(matches!(
|
||||
l.as_str(),
|
||||
"dl" | "rip" | "web" | "hd" | "hdr" | "dts" | "ac3" | "aac" | "remux" | "sample" | "eng" | "multi"
|
||||
) || l.chars().all(|c| c.is_ascii_digit())
|
||||
|| FORMAT.is_match(&l))
|
||||
};
|
||||
if let Some(g) = LEADING.captures(title).map(|c| c[1].trim().to_string()) {
|
||||
// `[1080p]` or `[HEVC]` at the front is a tag, not a group.
|
||||
if not_a_format(&g) && extract_quality(&g).is_none() {
|
||||
return Some(g);
|
||||
}
|
||||
}
|
||||
let mut t = title.trim().to_string();
|
||||
loop {
|
||||
let cut = TAIL_NOISE.replace(&t, "").trim_end().to_string();
|
||||
if cut == t {
|
||||
break;
|
||||
}
|
||||
t = cut;
|
||||
}
|
||||
TRAILING
|
||||
.captures(&t)
|
||||
.map(|c| c[1].to_string())
|
||||
.filter(|g| not_a_format(g))
|
||||
}
|
||||
|
||||
/// The cut of the work this release carries — `editions/<x>` members. A
|
||||
/// different cut has a different runtime, so these matter beyond a label
|
||||
/// (a subtitle synced to the theatrical cut drifts on the extended one).
|
||||
pub fn extract_editions(title: &str) -> BTreeSet<&'static str> {
|
||||
static EDITIONS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
[
|
||||
(r"(?i)\bextended\b", "extended"),
|
||||
(r"(?i)\bdirector'?s?[\s-]*cut\b", "directors-cut"),
|
||||
(r"(?i)\btheatrical\b", "theatrical"),
|
||||
(r"(?i)\bunrated\b", "unrated"),
|
||||
(r"(?i)\buncut\b", "uncut"),
|
||||
(r"(?i)\buncensored\b", "uncensored"),
|
||||
(r"(?i)\bimax\b", "imax"),
|
||||
(r"(?i)\bremaster(?:ed)?\b", "remastered"),
|
||||
(r"(?i)\bcriterion\b", "criterion"),
|
||||
(r"(?i)\banniversary\b", "anniversary"),
|
||||
(r"(?i)\bcollector'?s\b", "collectors"),
|
||||
(r"(?i)\bultimate[\s-]+(?:edition|cut)\b", "ultimate"),
|
||||
(r"(?i)\bspecial[\s-]+edition\b", "special"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(re, v)| (Regex::new(re).expect("edition re compiles"), v))
|
||||
.collect()
|
||||
});
|
||||
let t = spaced(title);
|
||||
EDITIONS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
|
||||
}
|
||||
|
||||
/// Properties of the encode itself — `flags/<x>` members: `repack` and
|
||||
/// `proper` (a fixed re-release that supersedes the first), `upscaled` (the
|
||||
/// resolution is not the source's: a 2160 upscale is not 4K), `regraded`.
|
||||
pub fn extract_flags(title: &str) -> BTreeSet<&'static str> {
|
||||
static FLAGS: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
|
||||
[
|
||||
(r"(?i)\b(?:repack\d?|rerip)\b", "repack"),
|
||||
(r"(?i)\bproper\b", "proper"),
|
||||
(r"(?i)\b(?:ai[\s-]*)?upscaled?\b", "upscaled"),
|
||||
(r"(?i)\bregraded?\b", "regraded"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(re, v)| (Regex::new(re).expect("flag re compiles"), v))
|
||||
.collect()
|
||||
});
|
||||
let t = spaced(title);
|
||||
FLAGS.iter().filter(|(re, _)| re.is_match(&t)).map(|(_, v)| *v).collect()
|
||||
}
|
||||
|
||||
/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` +
|
||||
/// `cleanSpace`, in that order.
|
||||
fn preclean(filename: &str) -> String {
|
||||
@@ -935,6 +1088,61 @@ pub fn clean_title(raw: &str) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn source_medium_reads_real_release_names() {
|
||||
let m = extract_source_medium;
|
||||
assert_eq!(m("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR"), Some("bluray-remux"));
|
||||
assert_eq!(m("The.Matrix.1999.1080p.BrRip.x264.YIFY.mp4"), Some("bluray"));
|
||||
assert_eq!(m("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv"), Some("web-dl"));
|
||||
assert_eq!(m("Show.S01E01.1080p.WEB.H264-GROUP"), Some("web-dl"));
|
||||
assert_eq!(m("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), Some("webrip"));
|
||||
assert_eq!(m("Movie 2023 HC HD-Rip 720p"), Some("hdrip"));
|
||||
assert_eq!(m("Some Show [2x10] HDTV"), Some("hdtv"));
|
||||
assert_eq!(m("Old.Film.1985.DVDRip.XviD"), Some("dvd"));
|
||||
assert_eq!(m("Old.Film.1985.DVD.REMUX"), Some("dvd"));
|
||||
assert_eq!(m("New.Film.2024.HDCAM.x264"), Some("cam"));
|
||||
assert_eq!(m("New Film 2024 CAM"), Some("cam"));
|
||||
assert_eq!(m("New.Film.2024.DVDSCR.x264"), Some("scr"));
|
||||
assert_eq!(m("New.Film.2024.TELESYNC"), Some("ts"));
|
||||
// AIOStreams' structured vocabulary goes through the same door.
|
||||
assert_eq!(m("BluRay REMUX"), Some("bluray-remux"));
|
||||
assert_eq!(m("WEB-DL"), Some("web-dl"));
|
||||
assert_eq!(m("TC"), Some("tc"));
|
||||
// The film "Cam" and a lower-case "ts" are not sources.
|
||||
assert_eq!(m("Cam (2018) 1080p"), None);
|
||||
assert_eq!(m("its a ts file"), None);
|
||||
assert_eq!(m("Movie 2019 1080p x264"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn release_group_is_the_leading_bracket_or_trailing_dash() {
|
||||
let g = |t: &str| extract_release_group(t);
|
||||
assert_eq!(g("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)").as_deref(), Some("yolerejiju"));
|
||||
assert_eq!(g("Breaking.Bad.S02E04.Giu.1080p.NF.WEB-DL.DDP5.1.AV1-G66.mkv").as_deref(), Some("G66"));
|
||||
assert_eq!(g("Game of Thrones S01E03 Lord Snow REPACK 1080p DTS-HD MA 5 1 AVC REMUX-FraMeSToR").as_deref(), Some("FraMeSToR"));
|
||||
assert_eq!(g("Oppenheimer.2023.1080p.BluRay.DD5.1.x264-GalaxyRG.mkv").as_deref(), Some("GalaxyRG"));
|
||||
assert_eq!(g("Movie.2020.1080p.WEB-DL.x264-NTb[rarbg]").as_deref(), Some("NTb"));
|
||||
assert_eq!(g("Breaking.Bad.S02E04.Down.1080p.BluRay.x264-OFT.mkv-xpost").as_deref(), Some("OFT"));
|
||||
assert_eq!(g("Movie 2020 1080p WEB-DL [TGx]").as_deref(), None, "WEB-DL is not a group called DL");
|
||||
assert_eq!(g("[1080p] Movie - 2020").as_deref(), None);
|
||||
assert_eq!(g("Movie.2020.1080p.x264").as_deref(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editions_and_flags() {
|
||||
let e = |t: &str| extract_editions(t).into_iter().collect::<Vec<_>>();
|
||||
assert_eq!(e("Aliens.1986.Directors.Cut.1080p.BluRay"), ["directors-cut"]);
|
||||
assert_eq!(e("LOTR.Fellowship.2001.EXTENDED.REMASTERED.2160p"), ["extended", "remastered"]);
|
||||
assert_eq!(e("Director's Cut"), ["directors-cut"]);
|
||||
assert_eq!(e("Ultimate Edition"), ["ultimate"]);
|
||||
assert_eq!(e("Movie.2019.1080p.WEB-DL"), Vec::<&str>::new());
|
||||
let f = |t: &str| extract_flags(t).into_iter().collect::<Vec<_>>();
|
||||
assert_eq!(f("Game of Thrones S01E03 Lord Snow REPACK 1080p"), ["repack"]);
|
||||
assert_eq!(f("Show.S01E01.PROPER.720p.HDTV"), ["proper"]);
|
||||
assert_eq!(f("Movie 1999 2160p AI Upscaled Regrade"), ["regraded", "upscaled"]);
|
||||
assert_eq!(f("The Proper Way 2020"), ["proper"], "a title word is indistinguishable — accepted noise");
|
||||
}
|
||||
|
||||
fn se(title: &str) -> (Option<String>, Option<String>) {
|
||||
let r = extract_season_episode(title);
|
||||
(r.season, r.episode)
|
||||
|
||||
+40
@@ -12,6 +12,10 @@
|
||||
//! With the `transport` feature it also carries the **transport-plugin**
|
||||
//! contract ([`transport`]): the same split, applied to meta-share — a
|
||||
//! protocol-agnostic hull and one sidecar per byte-moving protocol.
|
||||
//!
|
||||
//! [`beacon`] is beacon v2 local discovery. Both harnesses advertise through it;
|
||||
//! with `default-features = false, features = ["beacon"]` it is all a meta-*
|
||||
//! service pulls in.
|
||||
|
||||
// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it
|
||||
// directly rather than boxing on every embedded-DB call. Boxing each would be
|
||||
@@ -25,38 +29,74 @@
|
||||
/// `metacore-app` service the AppStore feeder apps ship on the shared network.
|
||||
pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000";
|
||||
|
||||
#[cfg(feature = "beacon")]
|
||||
pub mod beacon;
|
||||
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod budget;
|
||||
#[cfg(any(feature = "feeder", feature = "claims"))]
|
||||
pub mod claims;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod cache;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod common;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod config;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod domain;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod enrich;
|
||||
#[cfg(feature = "filename")]
|
||||
pub mod filename_meta;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod hash;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod lang;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod licence;
|
||||
/// Search-wire-transient `live/*` fields — always available, no dependencies.
|
||||
pub mod live;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod meta_core;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod plugin;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod query;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod query_eval;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod serve;
|
||||
#[cfg(feature = "transport")]
|
||||
pub mod transport;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub mod types;
|
||||
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use budget::RateBudget;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use config::{ConfigField, ConfigSchema, FieldKind};
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use licence::licence_from_url;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher};
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use meta_core::FeederStore;
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use plugin::{
|
||||
upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry,
|
||||
RedeemClaim,
|
||||
};
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter};
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use serve::{
|
||||
configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto,
|
||||
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
|
||||
RedeemsResponse,
|
||||
};
|
||||
#[cfg(any(feature = "feeder", feature = "claims"))]
|
||||
pub use claims::{ClaimValue, SignedClaim};
|
||||
#[cfg(feature = "claims")]
|
||||
pub use claims::{load_or_create_key, ClaimError, Signer};
|
||||
#[cfg(feature = "feeder")]
|
||||
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
||||
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
//! Search-wire-transient fields: facts a feeder sees at query time that are
|
||||
//! true only *now* — a torrent's seeders, its leechers — and so must never
|
||||
//! reach the replicated `/file/{cid}` hash (METADATA_KEYS rule #4).
|
||||
//!
|
||||
//! They travel under one reserved prefix, [`LIVE_PREFIX`], so every layer that
|
||||
//! persists a record strips the whole family without knowing each key:
|
||||
//!
|
||||
//! ```text
|
||||
//! live/seeders = "42" int-string
|
||||
//! live/leechers = "3" int-string
|
||||
//! live/at = "1791726110" unix seconds the feeder observed them
|
||||
//! ```
|
||||
//!
|
||||
//! A record without them means "unknown", never "zero": a replayed or
|
||||
//! persisted record has none, and that is the honest answer.
|
||||
//!
|
||||
//! The feeder SDK strips them in [`crate::meta_core::build_metadata_body`]
|
||||
//! (every feeder self-publish); meta-gateway and meta-search strip them on
|
||||
//! their own write paths (see METADATA_KEYS "Search-wire-transient fields").
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// The reserved prefix of the transient family.
|
||||
pub const LIVE_PREFIX: &str = "live/";
|
||||
|
||||
/// Swarm seeders, `int-string`.
|
||||
pub const LIVE_SEEDERS: &str = "live/seeders";
|
||||
/// Swarm leechers, `int-string`.
|
||||
pub const LIVE_LEECHERS: &str = "live/leechers";
|
||||
/// When the feeder observed the `live/*` values, unix seconds.
|
||||
pub const LIVE_AT: &str = "live/at";
|
||||
|
||||
/// The bare swarm counters some feeders wrote before the `live/*` family
|
||||
/// existed (tribler). Stripped with it: rule #4 bars them in any spelling.
|
||||
pub const LEGACY_SWARM_KEYS: [&str; 3] = ["seeders", "leechers", "peers"];
|
||||
|
||||
/// Whether a field is search-wire-transient and must not be persisted.
|
||||
pub fn is_transient(key: &str) -> bool {
|
||||
key.starts_with(LIVE_PREFIX) || LEGACY_SWARM_KEYS.contains(&key)
|
||||
}
|
||||
|
||||
/// Remove every transient field. Call on the copy about to be persisted, never
|
||||
/// on the record still bound for the search wire.
|
||||
pub fn strip_transient(fields: &mut BTreeMap<String, String>) {
|
||||
fields.retain(|k, _| !is_transient(k));
|
||||
}
|
||||
|
||||
/// The `live/at` value for a feeder observing counters right now.
|
||||
pub fn now_secs() -> String {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn strips_the_family_and_the_legacy_counters_only() {
|
||||
let mut f: BTreeMap<String, String> = [
|
||||
("live/seeders", "42"),
|
||||
("live/at", "1"),
|
||||
("seeders", "7"),
|
||||
("peers", "9"),
|
||||
("fileName", "x.mkv"),
|
||||
("livestream", "true"),
|
||||
("source/live", "true"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect();
|
||||
strip_transient(&mut f);
|
||||
let keys: Vec<&str> = f.keys().map(String::as_str).collect();
|
||||
assert_eq!(keys, ["fileName", "livestream", "source/live"]);
|
||||
}
|
||||
}
|
||||
+119
-21
@@ -12,10 +12,12 @@
|
||||
//! written by the feeder is indistinguishable from one the dispatcher wrote —
|
||||
//! the meta-share ingester, meta-watch, and ranking all behave the same.
|
||||
//!
|
||||
//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core
|
||||
//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the
|
||||
//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}`
|
||||
//! merges accumulate on the same `/file/{cid}` hash.
|
||||
//! Since meta-core v2.0.0 every write is a batch of **signed claims** to
|
||||
//! `POST /api/claims` (spec `docs/implementation/metacore-claims-model.md`):
|
||||
//! each field becomes one MMUR2 vote signed by this feeder's own key. Votes from
|
||||
//! different writers never overwrite each other — meta-core merges them — so the
|
||||
//! feeder's base write and the enrichment plugins' later claims accumulate on
|
||||
//! the same cid.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
@@ -23,6 +25,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use serde::Deserialize;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::claims::{now_ms, ClaimValue, SignedClaim, Signer};
|
||||
use crate::plugin::{HashKind, HashOutcome};
|
||||
use crate::types::{DiscoveryRecord, Hash};
|
||||
|
||||
@@ -80,6 +83,11 @@ pub fn build_metadata_body(
|
||||
) -> BTreeMap<String, String> {
|
||||
let mut body: BTreeMap<String, String> = BTreeMap::new();
|
||||
for (k, v) in &record.fields {
|
||||
// `live/*` (and the legacy bare swarm counters) are true only at query
|
||||
// time; a self-publishing feeder must not freeze them into the hash.
|
||||
if crate::live::is_transient(k) {
|
||||
continue;
|
||||
}
|
||||
if k.starts_with("cid_") && !v.is_empty() {
|
||||
body.insert(format!("cids/{v}"), "true".to_string());
|
||||
} else {
|
||||
@@ -90,22 +98,76 @@ pub fn build_metadata_body(
|
||||
body
|
||||
}
|
||||
|
||||
/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into
|
||||
/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`).
|
||||
pub async fn put_record(
|
||||
/// One per-claim verdict from `POST /api/claims`.
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct ClaimStatus {
|
||||
pub status: String,
|
||||
#[serde(default)]
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
/// `POST /api/claims` — submit already-signed claims (e.g. relayed verbatim).
|
||||
/// Fails on a non-2xx answer or when any claim is refused as `bad_signature` /
|
||||
/// `bad_request`; `stale` and `unchanged` are normal outcomes.
|
||||
pub async fn post_claims(
|
||||
http: &reqwest::Client,
|
||||
base_url: &str,
|
||||
cid: &str,
|
||||
body: &BTreeMap<String, String>,
|
||||
) -> anyhow::Result<()> {
|
||||
let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid);
|
||||
let resp = http.put(&url).json(body).send().await?;
|
||||
claims: &[SignedClaim],
|
||||
) -> anyhow::Result<Vec<ClaimStatus>> {
|
||||
if claims.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let url = format!("{}/api/claims", base_url.trim_end_matches('/'));
|
||||
let resp = http.post(&url).json(claims).send().await?;
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("meta-core PUT {url} returned {status}: {text}");
|
||||
anyhow::bail!("meta-core POST {url} returned {status}: {text}");
|
||||
}
|
||||
Ok(())
|
||||
let verdicts: Vec<ClaimStatus> = resp.json().await?;
|
||||
if let Some((i, bad)) = verdicts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find(|(_, v)| v.status == "bad_signature" || v.status == "bad_request")
|
||||
{
|
||||
anyhow::bail!(
|
||||
"meta-core refused claim {} ({}): {} {}",
|
||||
i,
|
||||
claims.get(i).map(|c| c.key.as_str()).unwrap_or("?"),
|
||||
bad.status,
|
||||
bad.reason.as_deref().unwrap_or("")
|
||||
);
|
||||
}
|
||||
Ok(verdicts)
|
||||
}
|
||||
|
||||
/// Sign every field of `body` as a text claim on `cid` (all at "now") and
|
||||
/// submit them. The claims-model replacement for the v1 `PUT /api/metadata`.
|
||||
pub async fn put_record(
|
||||
http: &reqwest::Client,
|
||||
base_url: &str,
|
||||
signer: &Signer,
|
||||
cid: &str,
|
||||
body: &BTreeMap<String, String>,
|
||||
) -> anyhow::Result<()> {
|
||||
let claims = signer.sign_fields(cid, body, now_ms());
|
||||
post_claims(http, base_url, &claims).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Sign retractions of `keys` on `cid` (this signer's own votes only).
|
||||
pub async fn retract_keys(
|
||||
http: &reqwest::Client,
|
||||
base_url: &str,
|
||||
signer: &Signer,
|
||||
cid: &str,
|
||||
keys: &[String],
|
||||
) -> anyhow::Result<()> {
|
||||
let ts = now_ms();
|
||||
let claims: Vec<SignedClaim> = keys
|
||||
.iter()
|
||||
.map(|k| signer.sign(cid, k, ClaimValue::Null, ts))
|
||||
.collect();
|
||||
post_claims(http, base_url, &claims).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`).
|
||||
@@ -157,27 +219,49 @@ pub struct FeederStore {
|
||||
http: reqwest::Client,
|
||||
meta_core_url: String,
|
||||
gateway_peer: String,
|
||||
signer: Signer,
|
||||
}
|
||||
|
||||
impl FeederStore {
|
||||
pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self {
|
||||
pub fn new(
|
||||
http: reqwest::Client,
|
||||
meta_core_url: String,
|
||||
gateway_peer: String,
|
||||
signer: Signer,
|
||||
) -> Self {
|
||||
FeederStore {
|
||||
http,
|
||||
meta_core_url: meta_core_url.trim_end_matches('/').to_string(),
|
||||
gateway_peer,
|
||||
signer,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build from the feeder env. `None` when `META_CORE_URL` is unset (the
|
||||
/// feeder then falls back to the legacy "return outcomes, gateway stores"
|
||||
/// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`.
|
||||
/// Build from the feeder env. `None` when `META_CORE_URL` is unset, or when
|
||||
/// no signing key is configured (`METAMESH_SIGNING_KEY` /
|
||||
/// `METAMESH_SIGNING_KEY_FILE`) — the feeder then falls back to the legacy
|
||||
/// "return outcomes, gateway stores" path. `gateway_peer` ←
|
||||
/// `META_GATEWAY_PEER_ID` / `HOSTNAME`.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let meta_core_url = env_nonempty("META_CORE_URL")?;
|
||||
let signer = match crate::claims::load_or_create_key() {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
warn!(target: "meta-feeder::store", error = %e,
|
||||
"META_CORE_URL set but no signing key; not self-publishing");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID")
|
||||
.or_else(|| env_nonempty("HOSTNAME"))
|
||||
.unwrap_or_else(|| "gateway-feeder".to_string());
|
||||
let http = crate::common::build_http_client(30, "meta-feeder-store", None);
|
||||
Some(Self::new(http, meta_core_url, gateway_peer))
|
||||
Some(Self::new(http, meta_core_url, gateway_peer, signer))
|
||||
}
|
||||
|
||||
/// The uid this store signs as.
|
||||
pub fn signer(&self) -> &Signer {
|
||||
&self.signer
|
||||
}
|
||||
|
||||
pub fn meta_core_url(&self) -> &str {
|
||||
@@ -257,7 +341,8 @@ impl FeederStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance.
|
||||
/// Metadata-only self-publish: the record's fields (+ provenance) as signed
|
||||
/// claims to `POST /api/claims`.
|
||||
async fn store_metadata_only(
|
||||
&self,
|
||||
upstream_id: &str,
|
||||
@@ -267,7 +352,7 @@ impl FeederStore {
|
||||
) -> anyhow::Result<()> {
|
||||
let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer);
|
||||
let body = build_metadata_body(record, &prov);
|
||||
put_record(&self.http, &self.meta_core_url, cid, &body).await
|
||||
put_record(&self.http, &self.meta_core_url, &self.signer, cid, &body).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,6 +385,7 @@ mod tests {
|
||||
("title".into(), "Big Buck Bunny".into()),
|
||||
("cid_btih_v1_file".into(), "bafkPERFILE".into()),
|
||||
]),
|
||||
claims: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -332,4 +418,16 @@ mod tests {
|
||||
);
|
||||
assert!(body.contains_key("provenance"));
|
||||
}
|
||||
|
||||
/// A self-publish never freezes swarm counters, in either spelling.
|
||||
#[test]
|
||||
fn body_drops_transient_fields() {
|
||||
let mut r = rec();
|
||||
for (k, v) in [("live/seeders", "42"), ("live/at", "1"), ("seeders", "7"), ("leechers", "2")] {
|
||||
r.fields.insert(k.into(), v.into());
|
||||
}
|
||||
let body = build_metadata_body(&r, &Provenance::now("tribler", "abc", "peer"));
|
||||
assert!(body.keys().all(|k| !crate::live::is_transient(k)), "{body:?}");
|
||||
assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,6 +100,10 @@ pub enum GatewaySearchEvent {
|
||||
record_id: String,
|
||||
set: BTreeMap<String, String>,
|
||||
remove: Vec<String>,
|
||||
/// Signed form of the patch: `set` as text claims, `remove` as
|
||||
/// retractions. Empty from a feeder (the gateway core signs).
|
||||
#[serde(default)]
|
||||
claims: Vec<crate::claims::SignedClaim>,
|
||||
},
|
||||
/// Retract a previously-emitted `Base` record entirely.
|
||||
Drop { record_id: String },
|
||||
|
||||
+45
-6
@@ -278,8 +278,15 @@ fn gateway_error_response(e: GatewayError) -> Response {
|
||||
// -------- handlers ---------------------------------------------------------
|
||||
|
||||
async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
|
||||
let mut plugins: Vec<PluginManifest> = state
|
||||
.plugins
|
||||
Json(build_manifest(&state.plugins, &state.version))
|
||||
}
|
||||
|
||||
/// The `/manifest` body — also what the beacon advertise's `rev` is a hash of.
|
||||
fn build_manifest(
|
||||
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
|
||||
version: &str,
|
||||
) -> ManifestResponse {
|
||||
let mut plugins: Vec<PluginManifest> = plugins
|
||||
.values()
|
||||
.map(|p| PluginManifest {
|
||||
id: p.upstream_id().to_string(),
|
||||
@@ -294,10 +301,27 @@ async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
|
||||
})
|
||||
.collect();
|
||||
plugins.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
Json(ManifestResponse {
|
||||
feeder_version: state.version.clone(),
|
||||
ManifestResponse {
|
||||
feeder_version: version.to_string(),
|
||||
plugins,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The feeder's beacon v2 resource: one `metamesh.feeder/<upstream_id>` cap per
|
||||
/// hosted plugin, the manifest endpoint, and `rev` = hash of the manifest.
|
||||
fn beacon_resource(
|
||||
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
|
||||
version: &str,
|
||||
listen_port: u16,
|
||||
) -> crate::beacon::Resource {
|
||||
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
|
||||
let m = build_manifest(plugins, version);
|
||||
let mut r = Resource::new("feeder", m.plugins.iter().map(|p| caps::feeder(&p.id)))
|
||||
.with_endpoint("http", advertise_url(listen_port))
|
||||
.with_endpoint("manifest", "/manifest");
|
||||
r.rev = Some(rev_of(&m));
|
||||
r.binds = binds_from_env();
|
||||
r
|
||||
}
|
||||
|
||||
async fn redeems(State(state): State<AppState>) -> Json<RedeemsResponse> {
|
||||
@@ -568,7 +592,8 @@ pub async fn serve_feeders(
|
||||
let state_dir = state_dir.into();
|
||||
let configured = configure_plugins(plugins, &state_dir)?;
|
||||
let version = env!("CARGO_PKG_VERSION").to_string();
|
||||
let app = router(configured, version, state_dir.clone());
|
||||
let advertised = configured.clone();
|
||||
let app = router(configured, version.clone(), state_dir.clone());
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(listen).await?;
|
||||
info!(
|
||||
@@ -576,6 +601,18 @@ pub async fn serve_feeders(
|
||||
%listen,
|
||||
"feeder listening"
|
||||
);
|
||||
// Beacon v2: advertise once the port is bound, so a consumer that hears us
|
||||
// can connect. The node lives for the process.
|
||||
let port = listen.port();
|
||||
let name = std::env::var("SERVICE_NAME").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| {
|
||||
let mut ids: Vec<&String> = advertised.keys().collect();
|
||||
ids.sort();
|
||||
ids.first().map_or_else(|| "meta-feeder".to_string(), |id| format!("meta-feeder-{id}"))
|
||||
});
|
||||
let beacon_version = version.clone();
|
||||
let _beacon = crate::beacon::advertise_plugin(&name, &version, move || {
|
||||
beacon_resource(&advertised, &beacon_version, port)
|
||||
});
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.map_err(anyhow::Error::from)
|
||||
@@ -762,6 +799,7 @@ mod tests {
|
||||
upstream_id: "stub".into(),
|
||||
record_id: "1".into(),
|
||||
fields: BTreeMap::new(),
|
||||
claims: Vec::new(),
|
||||
};
|
||||
let events = vec![
|
||||
GatewaySearchEvent::Base(rec),
|
||||
@@ -769,6 +807,7 @@ mod tests {
|
||||
record_id: "1".into(),
|
||||
set: BTreeMap::from([("title".to_string(), "X".to_string())]),
|
||||
remove: vec![],
|
||||
claims: vec![],
|
||||
},
|
||||
GatewaySearchEvent::Done,
|
||||
];
|
||||
|
||||
@@ -156,6 +156,25 @@ impl RemoteTransport {
|
||||
}
|
||||
}
|
||||
|
||||
/// `PUT /manifests/:cid`: hand a manifest-consuming plugin (contract 2) the
|
||||
/// manifest a pointer resolved to, after it answered `428` +
|
||||
/// [`super::dto::HDR_NEEDS_MANIFEST`].
|
||||
pub async fn push_manifest(&self, cid: &str, manifest: bytes::Bytes) -> Result<(), ApiError> {
|
||||
let r = self
|
||||
.http
|
||||
.put(self.url(&format!("/manifests/{cid}")))
|
||||
.body(manifest)
|
||||
.timeout(self.control_timeout)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| self.unreachable(e))?;
|
||||
if r.status().is_success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(error_from(r).await)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- typed helpers for extra routes ------------------------------------
|
||||
|
||||
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
|
||||
|
||||
+31
-2
@@ -9,8 +9,37 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
/// Bumped on any breaking change to the common routes below. The hull refuses
|
||||
/// a plugin whose `/manifest` reports a different major contract.
|
||||
pub const CONTRACT_VERSION: u32 = 1;
|
||||
/// a plugin whose `/manifest` reports a contract outside
|
||||
/// [`MIN_CONTRACT`]`..=`[`CONTRACT_VERSION`].
|
||||
///
|
||||
/// - **2**: a plugin that consumes a manifest (nzb) no longer pulls it from the
|
||||
/// hull. It answers `/raw` (and its verify route) with `428` +
|
||||
/// [`HDR_NEEDS_MANIFEST`] until the hull has pushed one with
|
||||
/// `PUT /manifests/:cid`.
|
||||
pub const CONTRACT_VERSION: u32 = 2;
|
||||
/// The oldest contract the hull still drives (contract-1 nzb plugins pull their
|
||||
/// manifest from `/internal/nzb/manifest/:cid`, kept for one release).
|
||||
pub const MIN_CONTRACT: u32 = 1;
|
||||
|
||||
/// `428` marker: this plugin needs the cid's manifest pushed before it can
|
||||
/// answer. The hull resolves the pointer, `PUT`s `/manifests/:cid`, and retries.
|
||||
pub const HDR_NEEDS_MANIFEST: &str = "x-metamesh-needs-manifest";
|
||||
|
||||
/// Why a pointer could not be resolved, on a non-2xx `/ipfs-tier/resolve`
|
||||
/// answer: [`RESOLVE_NO_GATEWAY`], [`RESOLVE_UNCLAIMED`], [`RESOLVE_QUOTA`],
|
||||
/// [`RESOLVE_NOT_FOUND`] or [`RESOLVE_UPSTREAM`].
|
||||
pub const HDR_RESOLVE_ERROR: &str = "x-metamesh-resolve-error";
|
||||
/// No gateway is known at all yet (discovery is eventual): retry.
|
||||
pub const RESOLVE_NO_GATEWAY: &str = "no-gateway";
|
||||
/// Gateways are known, but none claims this pointer's key.
|
||||
pub const RESOLVE_UNCLAIMED: &str = "unclaimed";
|
||||
/// The provider's quota is spent: retry after `Retry-After`.
|
||||
pub const RESOLVE_QUOTA: &str = "quota";
|
||||
/// Every claiming gateway answered not found (or nothing is stored and the
|
||||
/// pointer cannot be redeemed).
|
||||
pub const RESOLVE_NOT_FOUND: &str = "not-found";
|
||||
/// A gateway failed, or answered bytes that did not verify.
|
||||
pub const RESOLVE_UPSTREAM: &str = "upstream";
|
||||
|
||||
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
|
||||
pub const HDR_LANE: &str = "x-metamesh-lane";
|
||||
|
||||
@@ -39,10 +39,11 @@ use super::dto::Event;
|
||||
/// Default hull callback base, matching the dev and store compose service name.
|
||||
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
|
||||
|
||||
/// Where the hull got a manifest: the record's `manifest` pointer, or a
|
||||
/// gateway redeem.
|
||||
/// Where a pointer's bytes came from: `store` (already held here, named by the
|
||||
/// record's pointer field), `redeem` (a gateway), or `record` (contract-1
|
||||
/// callback naming for `store`).
|
||||
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
|
||||
/// The content cid of a `record`-sourced manifest.
|
||||
/// The content cid the pointer resolved to.
|
||||
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
|
||||
|
||||
/// `.nzb` bytes as the hull resolved them.
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
//! |---|---|
|
||||
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
|
||||
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
|
||||
//! | `GET resolve/:cid?hint=` → bytes | a [pointer](super::pointer)'s bytes: the stored copy `hint` names, else a gateway redeem (see below) |
|
||||
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
|
||||
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
|
||||
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
|
||||
@@ -40,6 +41,15 @@
|
||||
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
|
||||
//! a display name.
|
||||
|
||||
//! **Resolving a pointer** (`GET resolve/:cid`) is the one route that reaches
|
||||
//! outside the swarm: a pointer is answered only by a gateway holding the key,
|
||||
//! over plain HTTP at the gateway's base URL. Gateways come from config (the
|
||||
//! local one, pinned) and from the swarm directory; the fastest one whose redeem
|
||||
//! claim covers the pointer is asked first. `2xx` carries the bytes plus
|
||||
//! [`HDR_MANIFEST_SOURCE`](super::hull::HDR_MANIFEST_SOURCE) (`store` | `redeem`)
|
||||
//! and [`HDR_MANIFEST_CID`](super::hull::HDR_MANIFEST_CID); a failure carries
|
||||
//! [`HDR_RESOLVE_ERROR`](super::dto::HDR_RESOLVE_ERROR).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
|
||||
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
|
||||
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
|
||||
//! - [`pointer`]: which cids are pointers (resolved before their bytes exist).
|
||||
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
|
||||
//! - `testkit` (feature `testkit`): conformance checks for any implementation.
|
||||
//!
|
||||
@@ -42,6 +43,7 @@ pub mod ipfs;
|
||||
pub mod magic;
|
||||
pub mod nzb;
|
||||
pub mod plugin;
|
||||
pub mod pointer;
|
||||
pub mod range;
|
||||
pub mod serve;
|
||||
pub mod torrent;
|
||||
@@ -52,7 +54,7 @@ pub use client::RemoteTransport;
|
||||
pub use config::ConfigPlane;
|
||||
pub use dto::{
|
||||
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
|
||||
ReconcileRequest, CONTRACT_VERSION,
|
||||
ReconcileRequest, CONTRACT_VERSION, MIN_CONTRACT,
|
||||
};
|
||||
pub use error::ApiError;
|
||||
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
//! **Pointer cids** — cids that name something to fetch rather than hash it.
|
||||
//!
|
||||
//! A pointer's multihash is `identity`: the cid *contains* the address (an
|
||||
//! indexer host + release id, a provider source + file id), not a digest of the
|
||||
//! bytes. Bitswap cannot answer one, because there is no hash to verify against.
|
||||
//! It is resolved instead: by a manifest this peer already stored (the record's
|
||||
//! pointer field names its content cid), or by a **redeem** at a gateway whose
|
||||
//! feeder holds the credential for `key`.
|
||||
//!
|
||||
//! This module is the single place that knows which codecs are pointers, so the
|
||||
//! hull ("resolve it first") and the ipfs plugin ("here is how") agree.
|
||||
//!
|
||||
//! | codec | what it points at | redeem key | record field |
|
||||
//! |---|---|---|---|
|
||||
//! | `0x1005` nzb-release | a `.nzb` at an indexer | indexer host | `manifest` |
|
||||
//! | `0x1003` nzb-posting | a self-scanned `.nzb` | — (record only) | `manifest` |
|
||||
//! | `0x100A` provider-file | a file a provider holds | provider source | `file` |
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use super::cid::{parse_record_cid, read_uvarint};
|
||||
use super::nzb::{
|
||||
decode_nzb_release_cid, normalize_indexer_host, NZB_POSTING_CODEC, NZB_RELEASE_CODEC,
|
||||
};
|
||||
|
||||
/// MetaMesh-private multicodec for a provider-held file (`docs/cid-formats.md` §8).
|
||||
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
|
||||
|
||||
/// Redeem-claim codec names, as gateways advertise them in
|
||||
/// `GET /api/gateway/plugins` (`redeems[].codec`).
|
||||
pub const REDEEM_NZB_RELEASE: &str = "nzb-release";
|
||||
pub const REDEEM_PROVIDER_FILE: &str = "provider-file";
|
||||
|
||||
/// Record fields a redeem merges onto the pointer's record: the content cid of
|
||||
/// what it resolved to.
|
||||
pub const FIELD_MANIFEST: &str = "manifest";
|
||||
pub const FIELD_FILE: &str = "file";
|
||||
|
||||
/// Identity multihash code: the digest bytes are the payload verbatim.
|
||||
const IDENTITY_MULTIHASH: u64 = 0x00;
|
||||
|
||||
/// A decoded pointer.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Pointer {
|
||||
/// The redeem-claim codec, or `None` when no gateway can redeem it (a
|
||||
/// self-scanned posting resolves from the record alone).
|
||||
pub redeem: Option<RedeemKey>,
|
||||
/// The record field naming the resolved content cid.
|
||||
pub field: &'static str,
|
||||
/// What the resolved bytes are: a manifest a byte plugin consumes, or the
|
||||
/// file itself.
|
||||
pub kind: PointerKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RedeemKey {
|
||||
/// [`REDEEM_NZB_RELEASE`] or [`REDEEM_PROVIDER_FILE`].
|
||||
pub codec: &'static str,
|
||||
/// The indexer host / provider source a gateway's claim must cover.
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum PointerKind {
|
||||
/// A `.nzb`: handed to the nzb plugin, which fetches the bytes.
|
||||
NzbManifest,
|
||||
/// The file itself (a subtitle): served as is.
|
||||
File,
|
||||
}
|
||||
|
||||
/// Decode `cid` as a pointer.
|
||||
///
|
||||
/// - `None` — not a pointer: an ordinary content cid, or unparseable.
|
||||
/// - `Some(Err(_))` — the codec says pointer but the payload is malformed; the
|
||||
/// caller answers `400`.
|
||||
pub fn decode(cid: &str) -> Option<Result<Pointer>> {
|
||||
match codec_of(cid)? {
|
||||
NZB_RELEASE_CODEC => Some(decode_nzb_release(cid)),
|
||||
NZB_POSTING_CODEC => Some(Ok(Pointer {
|
||||
redeem: None,
|
||||
field: FIELD_MANIFEST,
|
||||
kind: PointerKind::NzbManifest,
|
||||
})),
|
||||
PROVIDER_FILE_CODEC => Some(decode_provider_file(cid).map(|(source, _id)| Pointer {
|
||||
redeem: Some(RedeemKey { codec: REDEEM_PROVIDER_FILE, key: source }),
|
||||
field: FIELD_FILE,
|
||||
kind: PointerKind::File,
|
||||
})),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Is `cid` a pointer (well-formed or not)?
|
||||
pub fn is_pointer(cid: &str) -> bool {
|
||||
decode(cid).is_some()
|
||||
}
|
||||
|
||||
fn decode_nzb_release(cid: &str) -> Result<Pointer> {
|
||||
let mscid = parse_record_cid(cid)?;
|
||||
let locator = decode_nzb_release_cid(&mscid)?;
|
||||
Ok(Pointer {
|
||||
redeem: Some(RedeemKey {
|
||||
codec: REDEEM_NZB_RELEASE,
|
||||
key: normalize_indexer_host(locator.authority()),
|
||||
}),
|
||||
field: FIELD_MANIFEST,
|
||||
kind: PointerKind::NzbManifest,
|
||||
})
|
||||
}
|
||||
|
||||
/// Decode a `provider-file` cid into `(source, id)`. Errors on a different
|
||||
/// codec or a malformed payload. Works on the raw multibase string because a
|
||||
/// long provider id can exceed the 64-byte multihash a typed cid parse allows.
|
||||
pub fn decode_provider_file(cid: &str) -> Result<(String, String)> {
|
||||
let bytes = raw_cid_bytes(cid).context("provider-file: not a base32 CIDv1")?;
|
||||
let mut at = 0usize;
|
||||
let next = |at: &mut usize, what: &str| -> Result<u64> {
|
||||
let (v, n) = read_uvarint(&bytes[*at..])
|
||||
.with_context(|| format!("provider-file: {what} varint"))?;
|
||||
*at += n;
|
||||
Ok(v)
|
||||
};
|
||||
anyhow::ensure!(next(&mut at, "version")? == 1, "provider-file: not a CIDv1");
|
||||
anyhow::ensure!(next(&mut at, "codec")? == PROVIDER_FILE_CODEC, "not a provider-file cid");
|
||||
let mh = next(&mut at, "multihash-code")?;
|
||||
anyhow::ensure!(
|
||||
mh == IDENTITY_MULTIHASH,
|
||||
"provider-file locator must use the identity multihash (0x00), got 0x{mh:x}"
|
||||
);
|
||||
let len = next(&mut at, "digest-length")?;
|
||||
anyhow::ensure!(
|
||||
(bytes.len() - at) as u64 == len,
|
||||
"provider-file digest length {len} != {} trailing byte(s)",
|
||||
bytes.len() - at
|
||||
);
|
||||
let source_len = next(&mut at, "source-length")?;
|
||||
let rest = &bytes[at..];
|
||||
let source_len = usize::try_from(source_len)
|
||||
.ok()
|
||||
.filter(|n| *n <= rest.len())
|
||||
.context("provider-file: source length overruns the digest")?;
|
||||
let (source, id) = rest.split_at(source_len);
|
||||
let source = std::str::from_utf8(source).context("provider-file source is not utf-8")?;
|
||||
let id = std::str::from_utf8(id).context("provider-file id is not utf-8")?;
|
||||
anyhow::ensure!(
|
||||
!source.is_empty() && !id.is_empty(),
|
||||
"provider-file locator needs both a source and an id"
|
||||
);
|
||||
Ok((source.to_string(), id.to_string()))
|
||||
}
|
||||
|
||||
/// The codec of a base32 CIDv1 string, without the 64-byte multihash limit.
|
||||
fn codec_of(cid: &str) -> Option<u64> {
|
||||
let bytes = raw_cid_bytes(cid)?;
|
||||
let (version, n) = read_uvarint(&bytes).ok()?;
|
||||
if version != 1 {
|
||||
return None;
|
||||
}
|
||||
read_uvarint(&bytes[n..]).ok().map(|(codec, _)| codec)
|
||||
}
|
||||
|
||||
/// Decode a `b`-prefixed lowercase base32 (RFC 4648, no padding) CID string.
|
||||
fn raw_cid_bytes(cid: &str) -> Option<Vec<u8>> {
|
||||
let body = cid.strip_prefix('b')?;
|
||||
let mut out = Vec::with_capacity(body.len() * 5 / 8);
|
||||
let (mut acc, mut bits) = (0u32, 0u32);
|
||||
for c in body.bytes() {
|
||||
let v = match c {
|
||||
b'a'..=b'z' => c - b'a',
|
||||
b'2'..=b'7' => c - b'2' + 26,
|
||||
_ => return None,
|
||||
} as u32;
|
||||
acc = (acc << 5) | v;
|
||||
bits += 5;
|
||||
if bits >= 8 {
|
||||
bits -= 8;
|
||||
out.push((acc >> bits) as u8);
|
||||
acc &= (1 << bits) - 1;
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The Ragna Crimson release from watch.nsl.sh (2026-10-04).
|
||||
const NZBGEEK: &str = "bagcsaabfcrqxa2jonz5gez3fmvvs42lomzxs6ylqngxuch22nhfhl4v4vmyhiegah2ka";
|
||||
|
||||
fn b32(bytes: &[u8]) -> String {
|
||||
const A: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
|
||||
let (mut acc, mut bits, mut s) = (0u32, 0u32, String::from("b"));
|
||||
for &b in bytes {
|
||||
acc = (acc << 8) | b as u32;
|
||||
bits += 8;
|
||||
while bits >= 5 {
|
||||
bits -= 5;
|
||||
s.push(A[((acc >> bits) & 31) as usize] as char);
|
||||
}
|
||||
acc &= (1 << bits) - 1;
|
||||
}
|
||||
if bits > 0 {
|
||||
s.push(A[((acc << (5 - bits)) & 31) as usize] as char);
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
fn provider_file(source: &str, id: &str) -> String {
|
||||
let mut digest = vec![source.len() as u8];
|
||||
digest.extend_from_slice(source.as_bytes());
|
||||
digest.extend_from_slice(id.as_bytes());
|
||||
// version 1, codec 0x100A (varint 0x8a 0x20), identity, length
|
||||
let mut wire = vec![0x01, 0x8a, 0x20, 0x00, digest.len() as u8];
|
||||
wire.extend_from_slice(&digest);
|
||||
b32(&wire)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_nzb_release_is_a_pointer_keyed_by_its_indexer_host() {
|
||||
let p = decode(NZBGEEK).unwrap().unwrap();
|
||||
assert_eq!(
|
||||
p.redeem,
|
||||
Some(RedeemKey { codec: REDEEM_NZB_RELEASE, key: "api.nzbgeek.info".into() })
|
||||
);
|
||||
assert_eq!(p.field, FIELD_MANIFEST);
|
||||
assert_eq!(p.kind, PointerKind::NzbManifest);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_provider_file_is_a_pointer_keyed_by_its_source() {
|
||||
let cid = provider_file("opensubtitles", "file:7061834");
|
||||
assert_eq!(
|
||||
decode_provider_file(&cid).unwrap(),
|
||||
("opensubtitles".into(), "file:7061834".into())
|
||||
);
|
||||
let p = decode(&cid).unwrap().unwrap();
|
||||
assert_eq!(p.redeem.unwrap().key, "opensubtitles");
|
||||
assert_eq!(p.field, FIELD_FILE);
|
||||
assert_eq!(p.kind, PointerKind::File);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_content_cid_is_not_a_pointer() {
|
||||
assert!(!is_pointer("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy"));
|
||||
assert!(!is_pointer("not a cid"));
|
||||
}
|
||||
}
|
||||
+32
-1
@@ -57,16 +57,47 @@ pub async fn serve_transport<P: TransportPlugin>(
|
||||
plugin: Arc<P>,
|
||||
listen: SocketAddr,
|
||||
) -> anyhow::Result<()> {
|
||||
let id = plugin.manifest().id;
|
||||
let manifest = plugin.manifest();
|
||||
let id = manifest.id.clone();
|
||||
let advertised = Arc::clone(&plugin);
|
||||
let app = router(plugin);
|
||||
let listener = tokio::net::TcpListener::bind(listen).await?;
|
||||
info!(%listen, plugin = %id, "transport plugin listening");
|
||||
// Beacon v2: advertise once bound; say `bye` on clean shutdown so the hull
|
||||
// drops us at once instead of after the liveness window.
|
||||
let port = listen.port();
|
||||
// `node.name` is a short machine name; the human description stays in the
|
||||
// manifest (`implementation`), which a consumer fetches over HTTP.
|
||||
let beacon_name = std::env::var("SERVICE_NAME")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.unwrap_or_else(|| format!("meta-transport-{}", manifest.id));
|
||||
let beacon = crate::beacon::advertise_plugin(&beacon_name, &manifest.version, move || {
|
||||
beacon_resource(advertised.as_ref(), port)
|
||||
});
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(shutdown_signal())
|
||||
.await?;
|
||||
if let Some(b) = beacon {
|
||||
b.bye().await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The transport's beacon v2 resource: id = manifest id, cap
|
||||
/// `metamesh.transport/<id>@<contract>`, `rev` = hash of the served manifest.
|
||||
fn beacon_resource<P: TransportPlugin + ?Sized>(plugin: &P, listen_port: u16) -> crate::beacon::Resource {
|
||||
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
|
||||
let mut m = plugin.manifest();
|
||||
m.config = plugin.config().is_some();
|
||||
let mut r = Resource::new(m.id.clone(), [caps::transport(&m.id, m.contract)])
|
||||
.with_endpoint("http", advertise_url(listen_port))
|
||||
.with_endpoint("manifest", "/manifest");
|
||||
r.rev = Some(rev_of(&m));
|
||||
r.binds = binds_from_env();
|
||||
r
|
||||
}
|
||||
|
||||
/// Resolves on SIGTERM (docker stop) or Ctrl-C.
|
||||
pub async fn shutdown_signal() {
|
||||
let ctrl_c = async {
|
||||
|
||||
@@ -49,6 +49,13 @@ pub struct DiscoveryRecord {
|
||||
pub upstream_id: String,
|
||||
pub record_id: String,
|
||||
pub fields: BTreeMap<String, String>,
|
||||
/// The record's MMUR2 claims (spec §10). Empty from a feeder that returns
|
||||
/// plain `fields` (the gateway core signs those as itself); filled by a
|
||||
/// self-publishing feeder or by the gateway before the record goes on the
|
||||
/// wire. `#[serde(default)]` keeps older feeders' JSON `/query` answers
|
||||
/// decoding.
|
||||
#[serde(default)] // never skip_serializing_if: bincode is positional
|
||||
pub claims: Vec<crate::claims::SignedClaim>,
|
||||
}
|
||||
|
||||
/// In-process error returned by `FeederPlugin` methods.
|
||||
@@ -102,6 +109,7 @@ mod tests {
|
||||
("title".to_string(), "AlphaFold".to_string()),
|
||||
("year".to_string(), "2021".to_string()),
|
||||
]),
|
||||
claims: Vec::new(),
|
||||
};
|
||||
let j = serde_json::to_string(&r).expect("serialize");
|
||||
let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize");
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"_comment": "MMUR2 golden vectors. Canonical copy: meta-core internal/claim/testdata. Do not edit by hand; see claim_test.go.",
|
||||
"privateKeyHex": "0101010101010101010101010101010101010101010101010101010101010101",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"vectors": [
|
||||
{
|
||||
"name": "text",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "title",
|
||||
"value": "The Matrix",
|
||||
"ts": 1793001234567,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656a546865204d61747269781b000001a17733a087",
|
||||
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3d2e560a2b61f4a81e320287cde753e1990d7949f4f518127f6505b8fb698ee8a8"
|
||||
},
|
||||
{
|
||||
"name": "empty-text",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "title",
|
||||
"value": "",
|
||||
"ts": 1,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a657469746c656001",
|
||||
"sigHex": "6eeb9ace8afe3987006c166eb0ec564b71e23d6a1c6211fdd55e8b4779c12dbb4f38c443faa3255c2ce7c6592e9d024e4ea57b5fa23446d1d4df647947e37642"
|
||||
},
|
||||
{
|
||||
"name": "unicode",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "titles/jpn/攻殻機動隊",
|
||||
"value": "true",
|
||||
"ts": 1793001234567,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a781a7469746c65732f6a706e2fe694bbe6aebbe6a99fe58b95e99a8a64747275651b000001a17733a087",
|
||||
"sigHex": "64ec6342c598782df278b49636a34da4f4c009423a584454a3284efad5426aa517d83873b09a4fdd3fd32a5e9a5fef0880126edd11db773045caeb6e634bc207"
|
||||
},
|
||||
{
|
||||
"name": "keyset-member",
|
||||
"cid": "bagacbabaec7v3",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "cids/bafkreih5kapbjzq",
|
||||
"value": "true",
|
||||
"ts": 23,
|
||||
"signedBytesHex": "4d4d5552320a856e6261676163626162616563377633782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a75636964732f6261666b72656968356b6170626a7a71647472756517",
|
||||
"sigHex": "8f707847895ccb522f1507dbab23330bd2c567d6fd1882df3bfae65bef6601024832f0b181146dcfb089aae91f5beac47da970373ab14370c4ac41410f3bf761"
|
||||
},
|
||||
{
|
||||
"name": "bytes",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "seek",
|
||||
"value": null,
|
||||
"valueBytesHex": "00ff10ab",
|
||||
"ts": 24,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647365656b4400ff10ab1818",
|
||||
"sigHex": "09671b27c03edd510029ec550b35e95e3900dd37a5d688ea04c3f0409979c7236bc4a1b5fa9d1b76265da85df24693a51f0ced528a6171c90a70b92db9f246e4"
|
||||
},
|
||||
{
|
||||
"name": "retraction",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "genres/Drama",
|
||||
"value": null,
|
||||
"ts": 255,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a6c67656e7265732f4472616d61f618ff",
|
||||
"sigHex": "d0c29d8edac4b78005eaf895cd9d534cb080b22ba9f7e79a5abf4d9bd906dc494464a2b388653452bca16e351132a9a92370953fa047e693459e55b7965449ac"
|
||||
},
|
||||
{
|
||||
"name": "long-value",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "plot/eng",
|
||||
"value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"ts": 65536,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a68706c6f742f656e6779012c6161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161616161611a00010000",
|
||||
"sigHex": "383b2e40f913eb5240e1eae2cfa392f967321d3701623c2b3df344984686d888198bce1c78847d1898b35f991ec3465c382c45e03b9a77f2ae7d1fd740391dd2"
|
||||
},
|
||||
{
|
||||
"name": "ts-64bit",
|
||||
"cid": "bafkreih5kapbjzq",
|
||||
"uid": "zvYNYVRtXSSDCi1rZtPP3ieuoh8cG5AscesGPYESa4VpJ",
|
||||
"key": "year",
|
||||
"value": "1999",
|
||||
"ts": 1099511627776,
|
||||
"signedBytesHex": "4d4d5552320a85706261666b72656968356b6170626a7a71782d7a76594e5956527458535344436931725a745050336965756f683863473541736365734750594553613456704a647965617264313939391b0000010000000000",
|
||||
"sigHex": "2b9b42ef4ebd7fc8d3ca0a514679ca4104f2e627da3a84c7b9664b27ebbdf73946cc37a04e4ea15dbed23a0e48e8797c92fcba5598b012305a314bf776398e0d"
|
||||
}
|
||||
],
|
||||
"highS": {
|
||||
"note": "same claim as `vector`, S replaced by n-S: valid ECDSA, but MMUR2 verifiers must refuse it",
|
||||
"vector": "text",
|
||||
"sigHex": "3ecac653f63cfc981564b5a5cc0db0459f2b4f19b810b6391d8f487216da0f3dd1a9f5d49e0b57e1cdfd783218ac1e65ad3592f1ba308dbc5acca59166a75899"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user