8 Commits
31 changed files with 7167 additions and 52 deletions
+49 -13
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "meta-feeder-sdk" name = "meta-feeder-sdk"
version = "0.1.0" version = "1.8.1"
edition = "2021" edition = "2021"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
@@ -31,33 +31,69 @@ publish = false
# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT # and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT
# of here so feeders that don't need them don't pay for them. # of here so feeders that don't need them don't pay for them.
[features]
default = ["feeder"]
# Everything a feeder sidecar (and the transport contract) builds on: the
# FeederPlugin trait, hashing, the redb cache, the axum harness. On by default
# so existing consumers build exactly what they always did.
feeder = [
"beacon", "filename",
"dep:axum", "dep:hyper", "dep:reqwest", "dep:base64", "dep:futures",
"dep:async-trait", "dep:anyhow", "dep:thiserror", "dep:sha2", "dep:bytes",
"dep:redb",
]
# Release-name parsing (`src/filename_meta.rs`: clean_title, season/episode,
# quality, codec). Needs only `regex`, so a client (meta-watch) can derive a
# display name from a raw release string without the feeder machinery.
filename = ["dep:regex"]
# Beacon v2 local discovery (`src/beacon/`). Needs only tokio/serde/tracing/libc,
# so a meta-* service takes the SDK with `default-features = false,
# features = ["beacon"]` and pulls in none of the feeder machinery.
beacon = ["dep:libc"]
# The meta-share transport-plugin contract (`src/transport/`): the hull's
# client, the plugins' axum harness, the focus replica, shared error/range
# types. Off by default so feeders build exactly what they always did.
transport = ["feeder", "dep:cid", "dep:tokio-util"]
# Conformance checks any transport implementation must pass.
testkit = ["transport"]
[dependencies] [dependencies]
# Async runtime + HTTP server (the serve_feeders harness) # Async runtime + HTTP server (the serve_feeders harness)
tokio = { version = "1.40", features = ["full"] } tokio = { version = "1.40", features = ["full"] }
axum = { version = "0.7", features = ["json"] } axum = { version = "0.7", features = ["json"], optional = true }
hyper = { version = "1", features = ["full"] } hyper = { version = "1", features = ["full"], optional = true }
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false, optional = true }
# Serialization # Serialization
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
base64 = "0.22" base64 = { version = "0.22", optional = true }
# Utilities # Utilities
futures = "0.3" futures = { version = "0.3", optional = true }
async-trait = "0.1" async-trait = { version = "0.1", optional = true }
anyhow = "1" anyhow = { version = "1", optional = true }
thiserror = "1" thiserror = { version = "1", optional = true }
tracing = "0.1" tracing = "0.1"
sha2 = "0.10" sha2 = { version = "0.10", optional = true }
bytes = "1" bytes = { version = "1", optional = true }
# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). # Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages).
regex = "1" regex = { version = "1", optional = true }
# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see # Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see
# cache.rs. Pure-Rust single-process embedded store. # cache.rs. Pure-Rust single-process embedded store.
redb = "2" redb = { version = "2", optional = true }
# CID parsing for the transport contract only (`transport::cid`). Optional so
# feeders don't pull it.
cid = { version = "0.11", optional = true }
# Raw sockets for `beacon` (per-interface multicast) and statvfs(2) for
# `transport::fsutil::available_bytes`.
libc = { version = "0.2", optional = true }
# `ReaderStream` for `transport::file`.
tokio-util = { version = "0.7", features = ["io"], optional = true }
[dev-dependencies] [dev-dependencies]
tempfile = "3" tempfile = "3"
tower = { version = "0.4", features = ["util"] }
tokio = { version = "1.40", features = ["full", "test-util"] } tokio = { version = "1.40", features = ["full", "test-util"] }
+77 -7
View File
@@ -17,7 +17,7 @@ bytes; it never joins the swarm.
```toml ```toml
[dependencies] [dependencies]
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v0.1.0" } meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.3.0" }
``` ```
Not on crates.io yet — the contract is still moving, so consumers pin a tag. Not on crates.io yet — the contract is still moving, so consumers pin a tag.
@@ -35,6 +35,42 @@ async fn main() -> anyhow::Result<()> {
A **service is a `Vec`** — grouping plugins into one binary is a deployment A **service is a `Vec`** — grouping plugins into one binary is a deployment
choice, not an architectural one. choice, not an architectural one.
### Discovery (1.6.0, scan 1.7.0)
`serve_feeders` and `serve_transport` advertise the process over **beacon v2**
(UDP multicast `239.255.99.1:9099`, spec: meta-root
`docs/project-architecture/beacon-v2.md`) as soon as the port is bound — a
feeder as `metamesh.feeder/<upstream_id>` (one cap per hosted plugin), a
transport as `metamesh.transport/<id>@<contract>` — with `endpoints.http`,
`endpoints.manifest = /manifest` and `rev` = a hash of the served manifest
(re-advertised when it changes). Nothing to call. Env:
| Env | Default | |
|---|---|---|
| `ENABLE_UDP_DISCOVERY` | `true` | `false` turns advertising off |
| `BEACON_ADVERTISE_URL` | `http://$HOSTNAME:<port>` | `endpoints.http` |
| `BEACON_BINDS` | unset | the one consumer instance allowed to list this plugin |
| `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS` | `239.255.99.1` / `9099` / `10000` | |
`beacon::scan(node, pattern, configured, wait)` (1.7.0) is what a consumer's
**Scan** button runs: probe everyone, wait, then classify every resource
matching `pattern` as `configured` / `addable` / `bound-elsewhere` against the
consumer's `(name, url)` list — the `ScanReport` JSON the shared
`<meta-beacon-scan>` element renders. Transports advertise as
`meta-transport-<id>` (`SERVICE_NAME` overrides), feeders as
`meta-feeder-<first upstream>`.
The `beacon` module is also usable on its own — the meta-* services depend on
the SDK for nothing else:
```toml
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0", default-features = false, features = ["beacon"] }
```
Features: `feeder` (default — the whole feeder harness; implies `beacon`),
`beacon` (tokio/serde/tracing/libc only), `transport` (implies `feeder`),
`testkit`.
## Writing a plugin ## Writing a plugin
Four required methods; everything else has a default: Four required methods; everything else has a default:
@@ -47,8 +83,32 @@ Four required methods; everything else has a default:
| `compute_outcomes` | resolve a `record_id` to `HashOutcome`s | `bytes: None` for metadata-only sources | | `compute_outcomes` | resolve a `record_id` to `HashOutcome`s | `bytes: None` for metadata-only sources |
Defaults you can override: `handle_query_stream`, `handle_fetch`, `health`, Defaults you can override: `handle_query_stream`, `handle_fetch`, `health`,
`served_file_types`, `served_content_kinds`, `config_schema`, `config_values`, `served_file_types`, `served_content_kinds`, `package`, `redeems`,
`get_blob`. `config_schema`, `config_values`, `get_blob`.
### Redeeming locators (1.3.0)
A plugin can **redeem** a locator family — turn a locator cid into the bytes it
names, usually spending a metered quota (a Newznab `.nzb` grab, an OpenSubtitles
download):
1. Return a `RedeemClaim` from `redeems()` — `RedeemClaim::nzb_release(hosts)` or
`RedeemClaim::provider_file(sources)` — reflecting the *current* config (no
credential → no claim). Served per plugin in `GET /manifest` and, for polling,
at `GET /redeems` → `{"redeems": {"<upstream_id>": [claim…]}}`.
2. Name a stable `package()` — the `/files/plugin/<package>/` folder the gateway
stores the bytes under.
3. In `compute_outcomes`, accept the locator cid as `record_id` and return ONE
`Sha2_256` outcome: `hash = hash::compute_ipfs_cid(&bytes)`, `bytes`,
`file_extension`, and optionally a `record` whose fields the gateway merges
onto the locator record. Return `NotFound` when the cid is not yours (e.g. no
key for that host) so the gateway can try the next claimer; `RateLimited` when
the quota is spent.
The gateway calls this only on a real play, one claimer at a time, and writes
the bytes' cid onto the locator record under the claim's `field` (`manifest` /
`file`), so each file's quota is spent once. Decode helpers:
`hash::decode_nzb_release_cid`, `hash::decode_provider_file_cid`, `hash::codec_of`.
### The routing gate ### The routing gate
@@ -101,6 +161,9 @@ into the bitswap blockstore.
The last two carry an identity multihash. Anything that dispatches on the The last two carry an identity multihash. Anything that dispatches on the
multihash instead of the **codec** will silently mis-rank them. multihash instead of the **codec** will silently mis-rank them.
`hash` also encodes/decodes the redeemable locators: `nzb-release` (`0x1005`)
and `provider-file` (`0x100A`, `(source, id)` — `docs/cid-formats.md` §8).
## Layout ## Layout
| module | what | | module | what |
@@ -114,6 +177,7 @@ multihash instead of the **codec** will silently mis-rank them.
| `hash` | the content-addressing families above | | `hash` | the content-addressing families above |
| `filename_meta` / `lang` | release-tag parsing helpers | | `filename_meta` / `lang` | release-tag parsing helpers |
| `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract | | `enrich`, `meta_core`, `common` | MetaMesh-internal; not part of the plugin contract |
| `beacon` | beacon v2 local discovery: `BeaconNode`, `Resource`, `cap_matches`, the MetaMesh `caps` vocabulary |
## Development ## Development
@@ -121,13 +185,19 @@ multihash instead of the **codec** will silently mis-rank them.
cargo build && cargo test cargo build && cargo test
``` ```
To work on the SDK from inside a feeder checkout without pushing a tag, add a To test a feeder against an SDK change **before its tag is pushed**: cargo will
gitignored `.cargo/config.toml` in the feeder: not resolve an unpushed tag, not even under a `[patch]` override ("failed to
find tag"). Build a scratch copy of the feeder (outside its git tree) with the
dependency swapped for a path, and run `cargo test` there:
```toml ```bash
paths = ["../meta-feeder-sdk"] mkdir -p /tmp/feeder && (cd meta-feeder-x && tar cf - --exclude=./target --exclude=./.git .) | (cd /tmp/feeder && tar xf -)
sed -i 's#meta-feeder-sdk = { git = .*#meta-feeder-sdk = { path = "/abs/path/to/meta-feeder-sdk" }#' /tmp/feeder/feeder-plugin/*/Cargo.toml
(cd /tmp/feeder && cargo test)
``` ```
Never commit the path form: each feeder repo is its own Docker build context.
## Context ## Context
Design rationale, the feeder roster, and the naming rules live in the meta-root: Design rationale, the feeder roster, and the naming rules live in the meta-root:
+165
View File
@@ -0,0 +1,165 @@
//! Beacon v2 — local resource advertise / discover over UDP multicast
//! (`239.255.99.1:9099`).
//!
//! Every meta-* service and every SDK-hosted plugin runs one [`BeaconNode`]: it
//! advertises this process's [`Resource`]s, each tagged with capability strings
//! (`metamesh.service/meta-sort`, `metamesh.transport/nzb@1`, …), and keeps a
//! live view of everyone else's, which consumers filter by capability
//! ([`cap_matches`]).
//!
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`
//! (the Go / TypeScript / Python ports are written against it too). This module
//! needs only the `beacon` feature, so a service can depend on the SDK with
//! `default-features = false, features = ["beacon"]` and pull in none of the
//! feeder machinery.
mod node;
mod proto;
mod scan;
pub use node::{enabled_from_env, hostname, BeaconNode, Config, SeenNode, SeenResource};
pub use scan::{classify, scan, suggested_name, Candidate, CandidateState, ScanReport, ScanSummary};
pub use proto::{
cap_matches, rev_of, Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT,
LIVENESS_FACTOR, MAX_DATAGRAM, PROTO, VERSION,
};
/// Capability vocabulary of the MetaMesh profile (spec Part 2).
pub mod caps {
/// meta-core; its resource carries `data.urls`.
pub const CORE: &str = "metamesh.core";
/// Prefix of `metamesh.service/<name>`.
pub const SERVICE_PREFIX: &str = "metamesh.service/";
/// Every service.
pub const ANY_SERVICE: &str = "metamesh.service/*";
/// Prefix of `metamesh.transport/<id>@<contract>`.
pub const TRANSPORT_PREFIX: &str = "metamesh.transport/";
/// Every transport plugin.
pub const ANY_TRANSPORT: &str = "metamesh.transport/*";
/// Prefix of `metamesh.feeder/<upstream_id>`.
pub const FEEDER_PREFIX: &str = "metamesh.feeder/";
/// Every feeder upstream.
pub const ANY_FEEDER: &str = "metamesh.feeder/*";
/// Prefix of `metamesh.enrich/<plugin_id>` (meta-sort enrichment plugins).
pub const ENRICH_PREFIX: &str = "metamesh.enrich/";
/// Every meta-sort enrichment plugin.
pub const ANY_ENRICH: &str = "metamesh.enrich/*";
pub fn service(name: &str) -> String {
format!("{SERVICE_PREFIX}{name}")
}
pub fn transport(id: &str, contract: u32) -> String {
format!("{TRANSPORT_PREFIX}{id}@{contract}")
}
pub fn feeder(upstream_id: &str) -> String {
format!("{FEEDER_PREFIX}{upstream_id}")
}
pub fn enrich(plugin_id: &str) -> String {
format!("{ENRICH_PREFIX}{plugin_id}")
}
}
/// The resource every meta-* service advertises: id `service`, cap
/// `metamesh.service/<name>`, `endpoints.ui` = its browser-facing URL.
pub fn service_resource(name: &str, ui_url: Option<&str>) -> Resource {
let mut r = Resource::new("service", [caps::service(name)]);
if let Some(u) = ui_url {
r = r.with_endpoint("ui", u);
}
r
}
/// `BEACON_ADVERTISE_URL`, else `http://$HOSTNAME:<port>` — what a plugin
/// advertises as `endpoints.http`.
pub fn advertise_url(listen_port: u16) -> String {
std::env::var("BEACON_ADVERTISE_URL")
.ok()
.map(|s| s.trim().trim_end_matches('/').to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("http://{}:{}", hostname(), listen_port))
}
/// `BEACON_BINDS`, when set.
pub fn binds_from_env() -> Option<String> {
std::env::var("BEACON_BINDS").ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
}
/// Advertise a plugin process: spawn a [`BeaconNode`] named `name` whose single
/// resource is whatever `build` returns, and re-run `build` every interval so a
/// changed manifest (`rev`) is re-advertised at once. Used by both SDK
/// harnesses; returns `None` (and logs) when discovery is disabled or the
/// socket can't be set up — a plugin keeps serving either way.
pub fn advertise_plugin<F>(name: &str, version: &str, build: F) -> Option<BeaconNode>
where
F: Fn() -> Resource + Send + Sync + 'static,
{
if !enabled_from_env() {
tracing::info!("beacon: disabled (ENABLE_UDP_DISCOVERY)");
return None;
}
let cfg = Config::from_env(name).version(version).resource(build());
let interval = cfg.interval;
let node = match BeaconNode::spawn(cfg) {
Ok(n) => n,
Err(e) => {
tracing::warn!(error = %e, "beacon: could not start; plugin will not be discoverable");
return None;
}
};
let refresher = node.clone();
tokio::spawn(async move {
let mut tick = tokio::time::interval(interval);
tick.tick().await;
loop {
tick.tick().await;
refresher.set_resources(vec![build()]).await;
}
});
Some(node)
}
/// The `/api/neighbors` JSON every service serves (spec Part 2, "HTTP
/// surface"). Each row is a [`SeenNode`] plus `caps` and a v1-compatible
/// `baseUrl` (the `metamesh.service/*` resource's `endpoints.ui`).
///
/// `all` returns every instance instead of one row per name; `cap` keeps only
/// nodes owning a resource that matches it.
pub fn neighbors_json(
node: Option<&BeaconNode>,
current: &str,
all: bool,
cap: Option<&str>,
) -> serde_json::Value {
let Some(node) = node else {
return serde_json::json!({
"current": current, "enabled": false, "count": 0, "neighbors": [],
});
};
let rows: Vec<serde_json::Value> = if all { node.nodes() } else { node.nodes_by_name() }
.into_iter()
.filter(|n| cap.map_or(true, |c| n.matches(c)))
.map(|n| neighbor_row(&n))
.collect();
serde_json::json!({
"current": current,
"enabled": true,
"count": rows.len(),
"self": neighbor_row(&node.self_node()),
"neighbors": rows,
})
}
fn neighbor_row(n: &SeenNode) -> serde_json::Value {
let mut v = serde_json::to_value(n).unwrap_or_default();
if let Some(obj) = v.as_object_mut() {
obj.insert("caps".into(), serde_json::json!(n.caps()));
let base = n.resource(caps::ANY_SERVICE).and_then(|r| r.endpoint("ui"));
if let Some(b) = base {
obj.insert("baseUrl".into(), serde_json::Value::String(b));
}
}
v
}
+719
View File
@@ -0,0 +1,719 @@
//! `BeaconNode` — advertise this process's resources and keep a live view of
//! everyone else's.
//!
//! Ported from meta-discovery v1's `meshdisco.rs`, keeping the two details that
//! fail *silently* when wrong:
//!
//! 1. **Per-interface fan-out.** A socket that joins with `INADDR_ANY` and sends
//! via the default route covers exactly one interface. `metashare-app` sits
//! on `pcs` *and* `metamesh-mesh`; `metawatch-*` on `metawatch` *and*
//! `metamesh-mesh`. We enumerate interfaces and join/send on each, using
//! `ip_mreqn`'s `imr_ifindex`.
//! 2. **Expiry at read time.** No reaper owns correctness; the announce loop's
//! prune only exists to fire change notifications for nodes that went quiet.
//!
//! Raw `libc` rather than `socket2` / `if-addrs`: every consumer already links
//! libc, and this is the whole of what we need from either crate.
use std::collections::HashMap;
use std::io;
use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4};
use std::os::unix::io::{AsRawFd, FromRawFd, RawFd};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde::Serialize;
use tokio::net::UdpSocket;
use tokio::sync::watch;
use tracing::{debug, info, warn};
use super::proto::{
Message, NodeInfo, Resource, DEFAULT_GROUP, DEFAULT_INTERVAL, DEFAULT_PORT, LIVENESS_FACTOR,
MAX_DATAGRAM, READ_BUFFER, TYPE_ADVERTISE, TYPE_BYE, TYPE_PROBE,
};
/// How a node is set up. Build with [`Config::new`] / [`Config::from_env`].
#[derive(Clone, Debug)]
pub struct Config {
pub node: NodeInfo,
pub resources: Vec<Resource>,
pub group: String,
pub port: u16,
pub interval: Duration,
}
impl Config {
/// Defaults: this host's name as the instance, the standard group/port,
/// a 10 s interval, no resources.
pub fn new(name: impl Into<String>) -> Self {
Self {
node: NodeInfo {
name: name.into(),
instance: hostname(),
version: None,
status: None,
},
resources: Vec::new(),
group: DEFAULT_GROUP.into(),
port: DEFAULT_PORT,
interval: DEFAULT_INTERVAL,
}
}
/// [`Config::new`] with `BEACON_GROUP` / `BEACON_PORT` / `BEACON_INTERVAL_MS`
/// applied.
pub fn from_env(name: impl Into<String>) -> Self {
let mut c = Self::new(name);
if let Some(g) = env_nonempty("BEACON_GROUP") {
c.group = g;
}
if let Some(p) = env_nonempty("BEACON_PORT").and_then(|v| v.parse().ok()) {
c.port = p;
}
if let Some(ms) = env_nonempty("BEACON_INTERVAL_MS").and_then(|v| v.parse::<u64>().ok()) {
if ms > 0 {
c.interval = Duration::from_millis(ms);
}
}
c
}
pub fn version(mut self, v: impl Into<String>) -> Self {
self.node.version = Some(v.into());
self
}
pub fn instance(mut self, i: impl Into<String>) -> Self {
self.node.instance = i.into();
self
}
pub fn resource(mut self, r: Resource) -> Self {
self.resources.push(r);
self
}
}
/// A node heard on the wire (or this node itself, via [`BeaconNode::self_node`]).
#[derive(Clone, Debug, Serialize)]
pub struct SeenNode {
#[serde(flatten)]
pub node: NodeInfo,
pub resources: Vec<Resource>,
/// Source address taken from the packet, never from the payload.
pub addr: String,
/// Unix seconds.
#[serde(rename = "lastSeen")]
pub last_seen: f64,
}
impl SeenNode {
/// Every cap of every resource, in order, deduplicated.
pub fn caps(&self) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for c in self.resources.iter().flat_map(|r| r.caps.iter()) {
if !out.contains(c) {
out.push(c.clone());
}
}
out
}
/// True if any resource matches `pattern`.
pub fn matches(&self, pattern: &str) -> bool {
self.resources.iter().any(|r| r.matches(pattern))
}
/// The first resource matching `pattern`.
pub fn resource(&self, pattern: &str) -> Option<&Resource> {
self.resources.iter().find(|r| r.matches(pattern))
}
}
/// One resource together with the node advertising it.
#[derive(Clone, Debug, Serialize)]
pub struct SeenResource {
pub node: NodeInfo,
pub resource: Resource,
pub addr: String,
#[serde(rename = "lastSeen")]
pub last_seen: f64,
}
struct Inner {
group: Ipv4Addr,
port: u16,
interval: Duration,
node: RwLock<NodeInfo>,
resources: RwLock<Vec<Resource>>,
socket: UdpSocket,
ifindexes: Vec<(String, u32)>,
seen: Mutex<HashMap<String, SeenNode>>,
changes: watch::Sender<u64>,
stopped: AtomicBool,
}
/// A running beacon v2 participant. Cheap to clone.
#[derive(Clone)]
pub struct BeaconNode {
inner: Arc<Inner>,
}
impl BeaconNode {
/// Bind, join the group on every eligible interface, and start the read and
/// advertise loops. Must be called inside a tokio runtime.
///
/// A join failure on one interface is logged and skipped rather than fatal —
/// a host that blocks multicast must degrade, not fail to boot.
pub fn spawn(cfg: Config) -> io::Result<BeaconNode> {
let group: Ipv4Addr = cfg.group.parse().map_err(|_| {
io::Error::new(io::ErrorKind::InvalidInput, format!("bad group {}", cfg.group))
})?;
let std_sock = bind_reuse(cfg.port)?;
set_multicast_ttl(&std_sock, 1)?;
let ifindexes = eligible_interfaces();
let mut joined = 0usize;
for (name, idx) in &ifindexes {
match join_group(&std_sock, group, *idx) {
Ok(()) => joined += 1,
Err(e) => warn!(iface = %name, error = %e, "beacon: join failed"),
}
}
info!(
group = %cfg.group, port = cfg.port, name = %cfg.node.name,
instance = %cfg.node.instance, resources = cfg.resources.len(),
joined, total = ifindexes.len(),
"beacon: listening"
);
std_sock.set_nonblocking(true)?;
let socket = UdpSocket::from_std(std_sock)?;
let (changes, _) = watch::channel(0u64);
let node = BeaconNode {
inner: Arc::new(Inner {
group,
port: cfg.port,
interval: cfg.interval,
node: RwLock::new(cfg.node),
resources: RwLock::new(cfg.resources),
socket,
ifindexes,
seen: Mutex::new(HashMap::new()),
changes,
stopped: AtomicBool::new(false),
}),
};
let reader = node.clone();
tokio::spawn(async move { reader.read_loop().await });
let advertiser = node.clone();
tokio::spawn(async move {
advertiser.advertise().await;
advertiser.probe(&[]).await;
let mut tick = tokio::time::interval(advertiser.inner.interval);
tick.tick().await; // the immediate first tick
loop {
tick.tick().await;
if advertiser.inner.stopped.load(Ordering::Relaxed) {
break;
}
advertiser.advertise().await;
advertiser.prune();
}
});
Ok(node)
}
/// The instance name this node advertises as.
pub fn instance(&self) -> String {
self.inner.node.read().unwrap().instance.clone()
}
/// Replace this node's resources and re-advertise at once (e.g. a manifest
/// `rev` changed). A no-op if nothing changed.
pub async fn set_resources(&self, resources: Vec<Resource>) {
{
let mut cur = self.inner.resources.write().unwrap();
if *cur == resources {
return;
}
*cur = resources;
}
self.advertise().await;
}
/// Set `node.status` (`starting` / `running`) and re-advertise.
pub async fn set_status(&self, status: &str) {
{
let mut n = self.inner.node.write().unwrap();
if n.status.as_deref() == Some(status) {
return;
}
n.status = Some(status.to_string());
}
self.advertise().await;
}
/// Multicast a probe. Every node owning a resource matching one of `want`
/// (every node, when `want` is empty) replies at once — what makes the
/// protocol usable on a boot path instead of waiting out an interval.
pub async fn probe(&self, want: &[String]) {
let msg = Message::probe(&self.instance(), want);
self.multicast(&msg).await;
}
/// Say goodbye: multicast `bye` and stop advertising. Call on clean
/// shutdown so neighbours drop us at once instead of after the TTL.
pub async fn bye(&self) {
if self.inner.stopped.swap(true, Ordering::Relaxed) {
return;
}
let node = self.inner.node.read().unwrap().clone();
self.multicast(&Message::bye(node)).await;
}
/// Fires (the counter increments) whenever the set of live nodes or any
/// node's resources change — new node, changed advertise, bye, expiry.
/// Consumers re-read [`BeaconNode::resources`] on each change.
pub fn subscribe(&self) -> watch::Receiver<u64> {
self.inner.changes.subscribe()
}
/// Every live node, sorted by (name, instance).
pub fn nodes(&self) -> Vec<SeenNode> {
let cutoff = self.cutoff();
let guard = self.inner.seen.lock().unwrap();
let mut out: Vec<SeenNode> = guard.values().filter(|n| n.last_seen >= cutoff).cloned().collect();
drop(guard);
out.sort_by(|a, b| a.node.name.cmp(&b.node.name).then(a.node.instance.cmp(&b.node.instance)));
out
}
/// One row per node name, most recently seen instance wins — the nav
/// menu's shape.
pub fn nodes_by_name(&self) -> Vec<SeenNode> {
let mut best: HashMap<String, SeenNode> = HashMap::new();
for n in self.nodes() {
match best.get(&n.node.name) {
Some(cur) if cur.last_seen >= n.last_seen => {}
_ => {
best.insert(n.node.name.clone(), n);
}
}
}
let mut out: Vec<SeenNode> = best.into_values().collect();
out.sort_by(|a, b| a.node.name.cmp(&b.node.name));
out
}
/// Every live resource matching `pattern`, with its node.
pub fn resources(&self, pattern: &str) -> Vec<SeenResource> {
self.nodes()
.into_iter()
.flat_map(|n| {
let SeenNode { node, resources, addr, last_seen } = n;
resources.into_iter().filter(|r| r.matches(pattern)).map(move |resource| SeenResource {
node: node.clone(),
resource,
addr: addr.clone(),
last_seen,
})
})
.collect()
}
/// This node as a row, so a UI can render itself without waiting for its
/// own echo.
pub fn self_node(&self) -> SeenNode {
SeenNode {
node: self.inner.node.read().unwrap().clone(),
resources: self.inner.resources.read().unwrap().clone(),
addr: String::new(),
last_seen: now_secs(),
}
}
// -- internals ----------------------------------------------------------
fn cutoff(&self) -> f64 {
now_secs() - self.inner.interval.as_secs_f64() * LIVENESS_FACTOR as f64
}
fn bump(&self) {
self.inner.changes.send_modify(|g| *g = g.wrapping_add(1));
}
/// Drop expired nodes; notify if any went.
fn prune(&self) {
let cutoff = self.cutoff();
let removed = {
let mut guard = self.inner.seen.lock().unwrap();
let before = guard.len();
guard.retain(|_, n| n.last_seen >= cutoff);
before != guard.len()
};
if removed {
self.bump();
}
}
fn advertise_msg(&self) -> Message {
Message::advertise(
self.inner.node.read().unwrap().clone(),
self.inner.resources.read().unwrap().clone(),
)
}
async fn advertise(&self) {
if self.inner.stopped.load(Ordering::Relaxed) {
return;
}
let msg = self.advertise_msg();
self.multicast(&msg).await;
}
fn encode(msg: &Message) -> Option<Vec<u8>> {
match serde_json::to_vec(msg) {
Ok(b) => {
if b.len() > MAX_DATAGRAM {
warn!(bytes = b.len(), max = MAX_DATAGRAM, "beacon: datagram over the fragmentation-safe size");
}
Some(b)
}
Err(e) => {
warn!(error = %e, "beacon: encode failed");
None
}
}
}
/// Writes once per interface. Letting the routing table choose would reach
/// exactly one network — the failure this protocol exists to avoid.
async fn multicast(&self, msg: &Message) {
let Some(body) = Self::encode(msg) else { return };
let dst = SocketAddr::V4(SocketAddrV4::new(self.inner.group, self.inner.port));
for (name, idx) in &self.inner.ifindexes {
if let Err(e) = set_multicast_if(&self.inner.socket, *idx) {
warn!(iface = %name, error = %e, "beacon: select interface failed");
continue;
}
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
warn!(iface = %name, error = %e, "beacon: send failed");
}
}
}
async fn reply_to(&self, dst: SocketAddr) {
let Some(body) = Self::encode(&self.advertise_msg()) else { return };
if let Err(e) = self.inner.socket.send_to(&body, dst).await {
debug!(error = %e, "beacon: reply failed");
}
}
async fn read_loop(self) {
let mut buf = vec![0u8; READ_BUFFER];
loop {
let (n, src) = match self.inner.socket.recv_from(&mut buf).await {
Ok(v) => v,
Err(e) => {
debug!(error = %e, "beacon: read error");
continue;
}
};
let Some(msg) = Message::parse(&buf[..n]) else { continue };
let me = self.instance();
match msg.msg_type.as_str() {
TYPE_PROBE => {
if msg.from.as_deref() == Some(me.as_str()) || self.inner.stopped.load(Ordering::Relaxed) {
continue;
}
let wanted = msg.want.is_empty()
|| self.inner.resources.read().unwrap().iter().any(|r| r.matches_any(&msg.want));
if wanted {
self.reply_to(src).await;
}
}
TYPE_ADVERTISE | TYPE_BYE => {
let Some(node) = msg.node else { continue };
if node.instance == me {
continue; // our own multicast echo
}
let changed = if msg.msg_type == TYPE_BYE {
self.inner.seen.lock().unwrap().remove(&node.instance).is_some()
} else {
let cutoff = self.cutoff();
let mut guard = self.inner.seen.lock().unwrap();
let changed = match guard.get(&node.instance) {
Some(prev) => {
prev.last_seen < cutoff || prev.node != node || prev.resources != msg.resources
}
None => true,
};
guard.insert(
node.instance.clone(),
SeenNode {
node,
resources: msg.resources,
addr: src.ip().to_string(),
last_seen: now_secs(),
},
);
changed
};
if changed {
self.bump();
}
}
_ => {}
}
}
}
}
// ---------------------------------------------------------------------------
// Socket plumbing.
// ---------------------------------------------------------------------------
fn bind_reuse(port: u16) -> io::Result<std::net::UdpSocket> {
unsafe {
let fd: RawFd = libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0);
if fd < 0 {
return Err(io::Error::last_os_error());
}
// Must be set before bind. Lets several listeners share the port on one
// host (a test, a sidecar, a beacon v1 responder in the same netns).
let one: libc::c_int = 1;
for opt in [libc::SO_REUSEADDR, libc::SO_REUSEPORT] {
libc::setsockopt(
fd,
libc::SOL_SOCKET,
opt,
&one as *const _ as *const libc::c_void,
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
);
}
let addr = libc::sockaddr_in {
sin_family: libc::AF_INET as libc::sa_family_t,
sin_port: port.to_be(),
sin_addr: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
sin_zero: [0; 8],
};
if libc::bind(
fd,
&addr as *const _ as *const libc::sockaddr,
std::mem::size_of::<libc::sockaddr_in>() as libc::socklen_t,
) < 0
{
let err = io::Error::last_os_error();
libc::close(fd);
return Err(err);
}
Ok(std::net::UdpSocket::from_raw_fd(fd))
}
}
fn set_multicast_ttl(sock: &std::net::UdpSocket, ttl: libc::c_int) -> io::Result<()> {
// TTL 1: link-local only, never routed off the segment.
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_MULTICAST_TTL,
&ttl as *const _ as *const libc::c_void,
std::mem::size_of::<libc::c_int>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
/// `ip_mreqn` — `imr_ifindex` is what makes this a per-interface join instead
/// of "whatever the routing table picks".
fn mreqn(group: Ipv4Addr, ifindex: u32) -> libc::ip_mreqn {
libc::ip_mreqn {
imr_multiaddr: libc::in_addr { s_addr: u32::from(group).to_be() },
imr_address: libc::in_addr { s_addr: libc::INADDR_ANY.to_be() },
imr_ifindex: ifindex as libc::c_int,
}
}
fn join_group(sock: &std::net::UdpSocket, group: Ipv4Addr, ifindex: u32) -> io::Result<()> {
let req = mreqn(group, ifindex);
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_ADD_MEMBERSHIP,
&req as *const _ as *const libc::c_void,
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
fn set_multicast_if(sock: &UdpSocket, ifindex: u32) -> io::Result<()> {
let req = mreqn(Ipv4Addr::UNSPECIFIED, ifindex);
let rc = unsafe {
libc::setsockopt(
sock.as_raw_fd(),
libc::IPPROTO_IP,
libc::IP_MULTICAST_IF,
&req as *const _ as *const libc::c_void,
std::mem::size_of::<libc::ip_mreqn>() as libc::socklen_t,
)
};
if rc < 0 {
return Err(io::Error::last_os_error());
}
Ok(())
}
/// Every interface except loopback, as (name, ifindex).
fn eligible_interfaces() -> Vec<(String, u32)> {
let mut out = Vec::new();
unsafe {
let list = libc::if_nameindex();
if list.is_null() {
warn!("beacon: could not enumerate interfaces");
return out;
}
let mut p = list;
while (*p).if_index != 0 && !(*p).if_name.is_null() {
let name = std::ffi::CStr::from_ptr((*p).if_name).to_string_lossy().into_owned();
if name != "lo" {
out.push((name, (*p).if_index));
}
p = p.add(1);
}
libc::if_freenameindex(list);
}
out
}
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
}
/// `$HOSTNAME`, else `/etc/hostname`, else `unknown`.
pub fn hostname() -> String {
env_nonempty("HOSTNAME")
.or_else(|| {
std::fs::read_to_string("/etc/hostname")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
})
.unwrap_or_else(|| "unknown".into())
}
pub(crate) fn now_secs() -> f64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs_f64())
.unwrap_or(0.0)
}
/// `ENABLE_UDP_DISCOVERY`, permissive: only `false`/`0`/`no`/`off` disable.
pub fn enabled_from_env() -> bool {
match std::env::var("ENABLE_UDP_DISCOVERY") {
Ok(v) => !matches!(v.trim().to_ascii_lowercase().as_str(), "false" | "0" | "no" | "off"),
Err(_) => true,
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Two nodes on loopback-only hosts can't multicast to each other, so
/// exercise the receive path directly by unicasting into a node's port.
async fn node_on(port: u16, instance: &str, resources: Vec<Resource>) -> BeaconNode {
let mut cfg = Config::new("test").instance(instance);
cfg.port = port;
cfg.interval = Duration::from_millis(200);
cfg.resources = resources;
BeaconNode::spawn(cfg).unwrap()
}
async fn send(port: u16, msg: &Message) {
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(&serde_json::to_vec(msg).unwrap(), ("127.0.0.1", port)).await.unwrap();
}
fn free_port() -> u16 {
std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
}
fn info(instance: &str) -> NodeInfo {
NodeInfo { name: "peer".into(), instance: instance.into(), ..Default::default() }
}
#[tokio::test]
async fn advertise_bye_and_expiry() {
let port = free_port();
let node = node_on(port, "me", vec![]).await;
let mut rx = node.subscribe();
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
send(port, &Message::advertise(info("peer-1"), vec![r])).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert_eq!(node.resources("metamesh.transport/*").len(), 1);
assert!(node.resources("metamesh.core").is_empty());
send(port, &Message::bye(info("peer-1"))).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert!(node.nodes().is_empty());
send(port, &Message::advertise(info("peer-2"), vec![])).await;
tokio::time::timeout(Duration::from_secs(2), rx.changed()).await.unwrap().unwrap();
assert_eq!(node.nodes().len(), 1);
// 3 × 200 ms liveness, then the prune on the next tick notifies.
tokio::time::timeout(Duration::from_secs(3), rx.changed()).await.unwrap().unwrap();
assert!(node.nodes().is_empty());
}
#[tokio::test]
async fn own_echo_and_v1_ignored() {
let port = free_port();
let node = node_on(port, "me", vec![]).await;
send(port, &Message::advertise(info("me"), vec![])).await;
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(br#"{"type":"announce","name":"x","tools":[]}"#, ("127.0.0.1", port)).await.unwrap();
s.send_to(br#"{"v":1,"type":"announce","name":"meta-sort","instance":"a"}"#, ("127.0.0.1", port))
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(150)).await;
assert!(node.nodes().is_empty());
}
#[tokio::test]
async fn probe_reply_honours_want() {
let port = free_port();
let _node = node_on(port, "me", vec![Resource::new("service", ["metamesh.service/meta-sort"])]).await;
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
let mut buf = vec![0u8; READ_BUFFER];
let miss = Message::probe("cli", &["metamesh.core".to_string()]);
s.send_to(&serde_json::to_vec(&miss).unwrap(), ("127.0.0.1", port)).await.unwrap();
assert!(tokio::time::timeout(Duration::from_millis(300), s.recv_from(&mut buf)).await.is_err());
let hit = Message::probe("cli", &["metamesh.service/*".to_string()]);
s.send_to(&serde_json::to_vec(&hit).unwrap(), ("127.0.0.1", port)).await.unwrap();
let (n, _) = tokio::time::timeout(Duration::from_secs(2), s.recv_from(&mut buf)).await.unwrap().unwrap();
let m = Message::parse(&buf[..n]).unwrap();
assert_eq!(m.msg_type, TYPE_ADVERTISE);
assert_eq!(m.node.unwrap().instance, "me");
}
}
+312
View File
@@ -0,0 +1,312 @@
//! Beacon v2 wire types and capability matching.
//!
//! The normative spec is meta-root's `docs/project-architecture/beacon-v2.md`.
use std::collections::BTreeMap;
use std::hash::{Hash, Hasher};
use std::time::Duration;
use serde::{Deserialize, Serialize};
pub const PROTO: &str = "beacon";
pub const VERSION: u8 = 2;
/// Beacon v1's group and port, on purpose — one well-known endpoint for every
/// kind of local resource. v1 and v2 coexist because v2 never sends v1's
/// `discovery` / `announce` types.
pub const DEFAULT_GROUP: &str = "239.255.99.1";
pub const DEFAULT_PORT: u16 = 9099;
pub const DEFAULT_INTERVAL: Duration = Duration::from_secs(10);
/// A node unheard for `interval * this` is dropped (evaluated at read time).
pub const LIVENESS_FACTOR: u32 = 3;
/// Receive buffer. Senders stay under [`MAX_DATAGRAM`].
pub const READ_BUFFER: usize = 8192;
/// Above this a datagram risks IP fragmentation; the sender logs a warning.
pub const MAX_DATAGRAM: usize = 1400;
pub const TYPE_PROBE: &str = "probe";
pub const TYPE_ADVERTISE: &str = "advertise";
pub const TYPE_BYE: &str = "bye";
/// Who is advertising.
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct NodeInfo {
/// What the software is (`meta-sort`, `meta-transport-nzb`).
pub name: String,
/// Unique per running copy — the container hostname.
pub instance: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
/// `starting` | `running`. Absent reads as `running`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub status: Option<String>,
}
/// One advertised resource.
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub struct Resource {
/// Unique within the node.
pub id: String,
pub caps: Vec<String>,
/// Well-known names: `http`, `ui`, `manifest`, `mcp`. A value starting with
/// `/` is relative to `http` — see [`Resource::endpoint`].
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub endpoints: BTreeMap<String, String>,
/// Opaque revision of the resource's details (e.g. its manifest).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rev: Option<String>,
/// The one consumer instance this resource belongs to. Absent = anyone.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub binds: Option<String>,
/// Profile-defined, per capability.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub data: Option<serde_json::Value>,
}
impl Resource {
pub fn new(id: impl Into<String>, caps: impl IntoIterator<Item = impl Into<String>>) -> Self {
Self {
id: id.into(),
caps: caps.into_iter().map(Into::into).collect(),
..Default::default()
}
}
pub fn with_endpoint(mut self, name: impl Into<String>, url: impl Into<String>) -> Self {
let url = url.into();
if !url.is_empty() {
self.endpoints.insert(name.into(), url);
}
self
}
/// True if any of this resource's caps matches `pattern`.
pub fn matches(&self, pattern: &str) -> bool {
self.caps.iter().any(|c| cap_matches(pattern, c))
}
/// True if any cap matches any of `patterns`. An empty list matches.
pub fn matches_any(&self, patterns: &[String]) -> bool {
patterns.is_empty() || patterns.iter().any(|p| self.matches(p))
}
/// `binds` is absent, or names `instance`.
pub fn usable_by(&self, instance: &str) -> bool {
self.binds.as_deref().map_or(true, |b| b == instance)
}
/// The named endpoint as an absolute URL: a value starting with `/` is
/// joined onto `endpoints.http`.
pub fn endpoint(&self, name: &str) -> Option<String> {
let v = self.endpoints.get(name)?;
if v.starts_with('/') {
let base = self.endpoints.get("http")?;
Some(format!("{}{}", base.trim_end_matches('/'), v))
} else {
Some(v.clone())
}
}
}
/// Every v2 datagram.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Message {
pub proto: String,
pub v: u8,
#[serde(rename = "type")]
pub msg_type: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub node: Option<NodeInfo>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub resources: Vec<Resource>,
/// Probe only: informational sender instance.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub from: Option<String>,
/// Probe only: reply only if a resource matches one of these.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub want: Vec<String>,
}
impl Message {
fn base(msg_type: &str) -> Self {
Self {
proto: PROTO.into(),
v: VERSION,
msg_type: msg_type.into(),
node: None,
resources: Vec::new(),
from: None,
want: Vec::new(),
}
}
pub fn probe(from: &str, want: &[String]) -> Self {
Self {
from: Some(from.to_string()),
want: want.to_vec(),
..Self::base(TYPE_PROBE)
}
}
pub fn advertise(node: NodeInfo, resources: Vec<Resource>) -> Self {
Self {
node: Some(node),
resources,
..Self::base(TYPE_ADVERTISE)
}
}
pub fn bye(node: NodeInfo) -> Self {
Self {
node: Some(node),
..Self::base(TYPE_BYE)
}
}
/// Decode a datagram, returning `None` for anything that is not a
/// well-formed beacon v2 message — beacon v1 traffic (no envelope), other
/// versions, unknown types, or an advertise/bye without a node. The group
/// is shared, so all of that is silently ignored.
pub fn parse(bytes: &[u8]) -> Option<Self> {
let m: Message = serde_json::from_slice(bytes).ok()?;
if m.proto != PROTO || m.v != VERSION {
return None;
}
match m.msg_type.as_str() {
TYPE_PROBE => Some(m),
TYPE_ADVERTISE | TYPE_BYE => {
let n = m.node.as_ref()?;
if n.instance.is_empty() || n.name.is_empty() {
return None;
}
Some(m)
}
_ => None,
}
}
}
/// Split `base@contract`. A non-numeric or missing contract is `None`.
fn split_contract(s: &str) -> (&str, Option<&str>) {
match s.rsplit_once('@') {
Some((base, c)) => (base, Some(c)),
None => (s, None),
}
}
/// Does capability `cap` satisfy `pattern`?
///
/// - `*` matches everything.
/// - `@N` on the pattern must equal the cap's contract; without one the cap's
/// contract is ignored.
/// - `x/*` matches any variant of `x`; otherwise the bases must be equal.
pub fn cap_matches(pattern: &str, cap: &str) -> bool {
if pattern == "*" {
return true;
}
let (pbase, pcontract) = split_contract(pattern);
let (cbase, ccontract) = split_contract(cap);
if let Some(pc) = pcontract {
if ccontract != Some(pc) {
return false;
}
}
match pbase.strip_suffix("/*") {
Some(prefix) => cbase
.strip_prefix(prefix)
.is_some_and(|rest| rest.starts_with('/') && rest.len() > 1),
None => pbase == cbase,
}
}
/// A short opaque revision for any serialisable value — what a harness puts in
/// [`Resource::rev`] so consumers re-fetch a manifest only when it changed.
/// Stable for a given build; not cryptographic and not meant to be.
pub fn rev_of<T: Serialize>(value: &T) -> String {
let bytes = serde_json::to_vec(value).unwrap_or_default();
let mut h = std::collections::hash_map::DefaultHasher::new();
bytes.hash(&mut h);
format!("{:016x}", h.finish())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn wildcard_matches_any_variant_only() {
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/nzb@1"));
assert!(cap_matches("metamesh.transport/*", "metamesh.transport/ipfs"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transportx/a"));
assert!(!cap_matches("metamesh.transport/*", "metamesh.transport/"));
assert!(cap_matches("*", "anything@3"));
}
#[test]
fn contract_rules() {
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@1"));
assert!(cap_matches("metamesh.transport/nzb", "metamesh.transport/nzb@2"));
assert!(cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@2"));
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb@1"));
assert!(!cap_matches("metamesh.transport/nzb@2", "metamesh.transport/nzb"));
assert!(cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@1"));
assert!(!cap_matches("metamesh.transport/*@1", "metamesh.transport/nzb@2"));
}
#[test]
fn exact_match() {
assert!(cap_matches("mcp", "mcp"));
assert!(!cap_matches("mcp", "mcp/x"));
assert!(!cap_matches("metamesh.core", "metamesh.core2"));
assert!(!cap_matches("metamesh.transport/nzb", "metamesh.transport/ipfs@1"));
}
#[test]
fn parse_rejects_v1_and_foreign_traffic() {
// beacon v1 aggregator probe / server announce
assert!(Message::parse(br#"{"type":"discovery"}"#).is_none());
assert!(Message::parse(br#"{"type":"announce","name":"x","tools":[]}"#).is_none());
// meta-discovery v1
assert!(Message::parse(br#"{"v":1,"type":"announce","name":"meta-sort"}"#).is_none());
// wrong version / unknown type / not json
assert!(Message::parse(br#"{"proto":"beacon","v":3,"type":"probe"}"#).is_none());
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"announce"}"#).is_none());
assert!(Message::parse(b"\x00\x01garbage").is_none());
// advertise without a node
assert!(Message::parse(br#"{"proto":"beacon","v":2,"type":"advertise"}"#).is_none());
}
#[test]
fn roundtrip_advertise() {
let r = Resource::new("nzb", ["metamesh.transport/nzb@1"])
.with_endpoint("http", "http://metashare-nzb:3000")
.with_endpoint("manifest", "/manifest");
let m = Message::advertise(
NodeInfo { name: "meta-transport-nzb".into(), instance: "metashare-nzb".into(), ..Default::default() },
vec![r],
);
let bytes = serde_json::to_vec(&m).unwrap();
let back = Message::parse(&bytes).unwrap();
assert_eq!(back.msg_type, TYPE_ADVERTISE);
let r = &back.resources[0];
assert_eq!(r.endpoint("manifest").as_deref(), Some("http://metashare-nzb:3000/manifest"));
assert!(r.matches("metamesh.transport/*"));
}
#[test]
fn binds() {
let mut r = Resource::new("nzb", ["metamesh.transport/nzb@1"]);
assert!(r.usable_by("anyone"));
r.binds = Some("metashare-app".into());
assert!(r.usable_by("metashare-app"));
assert!(!r.usable_by("metawatch-share"));
}
#[test]
fn probe_filter() {
let r = Resource::new("service", ["metamesh.service/meta-sort"]);
assert!(r.matches_any(&[]));
assert!(r.matches_any(&["metamesh.service/*".into()]));
assert!(!r.matches_any(&["metamesh.core".into()]));
}
}
+234
View File
@@ -0,0 +1,234 @@
//! On-demand discovery scan — what a consumer's "Scan" button runs.
//!
//! Probe everyone, wait a moment for the replies, then classify every resource
//! matching the consumer's capability pattern against what the consumer has
//! already configured. The report is the same JSON in every consumer
//! (meta-share, meta-gateway, meta-sort), rendered by the shared
//! `<meta-beacon-scan>` element.
use std::time::{Duration, Instant};
use serde::Serialize;
use super::node::{BeaconNode, SeenResource};
/// How a discovered resource relates to this consumer.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "kebab-case")]
pub enum CandidateState {
/// Its URL is already in the consumer's list.
Configured,
/// Can be added.
Addable,
/// `binds` names another consumer; shown but not addable.
BoundElsewhere,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Candidate {
pub instance: String,
pub name: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub version: Option<String>,
pub resource_id: String,
pub caps: Vec<String>,
/// `endpoints.http` — what the consumer would register.
pub url: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub binds: Option<String>,
pub state: CandidateState,
/// The list entry that already holds this URL (state `configured`).
#[serde(skip_serializing_if = "Option::is_none")]
pub configured_as: Option<String>,
/// A list-safe name to register it under.
pub suggested_name: String,
}
#[derive(Clone, Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ScanSummary {
/// Every node heard on the segment, whatever it advertises.
pub nodes: usize,
/// Resources matching the consumer's pattern.
pub capable: usize,
pub configured: usize,
pub bound_elsewhere: usize,
pub addable: usize,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ScanReport {
pub cap: String,
/// This consumer's instance (what `binds` is compared against).
#[serde(rename = "self")]
pub self_instance: String,
pub duration_ms: u64,
pub summary: ScanSummary,
pub candidates: Vec<Candidate>,
}
fn norm(u: &str) -> String {
u.trim().trim_end_matches('/').to_string()
}
/// Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules.
pub fn suggested_name(instance: &str) -> String {
let s: String = instance
.trim()
.to_ascii_lowercase()
.chars()
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '-' })
.collect();
if s.is_empty() { "plugin".into() } else { s }
}
/// Pure classification, for tests and for callers that already hold a view.
/// `configured` is `(name, url)` of every entry the consumer has.
pub fn classify(
pattern: &str,
self_instance: &str,
nodes: usize,
resources: Vec<SeenResource>,
configured: &[(String, String)],
) -> ScanReport {
let mut summary = ScanSummary { nodes, ..Default::default() };
let mut candidates = Vec::new();
for r in resources {
// A resource without an http endpoint cannot be registered anywhere.
let Some(url) = r.resource.endpoint("http") else { continue };
let url = norm(&url);
summary.capable += 1;
let configured_as = configured.iter().find(|(_, u)| norm(u) == url).map(|(n, _)| n.clone());
let state = if configured_as.is_some() {
summary.configured += 1;
CandidateState::Configured
} else if !r.resource.usable_by(self_instance) {
summary.bound_elsewhere += 1;
CandidateState::BoundElsewhere
} else {
summary.addable += 1;
CandidateState::Addable
};
candidates.push(Candidate {
suggested_name: suggested_name(&r.node.instance),
instance: r.node.instance,
name: r.node.name,
version: r.node.version,
resource_id: r.resource.id,
caps: r.resource.caps,
url,
binds: r.resource.binds,
state,
configured_as,
});
}
// Addable first (what the operator came for), then configured, then the rest.
let rank = |s: CandidateState| match s {
CandidateState::Addable => 0,
CandidateState::Configured => 1,
CandidateState::BoundElsewhere => 2,
};
candidates.sort_by(|a, b| rank(a.state).cmp(&rank(b.state)).then(a.instance.cmp(&b.instance)));
ScanReport {
cap: pattern.to_string(),
self_instance: self_instance.to_string(),
duration_ms: 0,
summary,
candidates,
}
}
/// Probe every node, wait `wait` for the replies, then classify what matches
/// `pattern`. A node that stays silent through the wait but is still inside
/// its liveness window is counted too — the view is "who is here", not "who
/// answered this probe".
pub async fn scan(
node: &BeaconNode,
pattern: &str,
configured: &[(String, String)],
wait: Duration,
) -> ScanReport {
let started = Instant::now();
node.probe(&[]).await;
tokio::time::sleep(wait).await;
let mut report = classify(
pattern,
&node.instance(),
node.nodes().len(),
node.resources(pattern),
configured,
);
report.duration_ms = started.elapsed().as_millis() as u64;
report
}
#[cfg(test)]
mod tests {
use super::*;
use crate::beacon::{NodeInfo, Resource};
fn seen(instance: &str, cap: &str, url: Option<&str>, binds: Option<&str>) -> SeenResource {
let mut r = Resource::new("x", [cap]);
if let Some(u) = url {
r = r.with_endpoint("http", u);
}
r.binds = binds.map(Into::into);
SeenResource {
node: NodeInfo { name: "plugin".into(), instance: instance.into(), ..Default::default() },
resource: r,
addr: "10.0.0.1".into(),
last_seen: 0.0,
}
}
#[test]
fn classifies_configured_addable_and_bound_elsewhere() {
let resources = vec![
seen("metashare-nzb", "metamesh.transport/nzb@1", Some("http://metashare-nzb:3000/"), Some("metashare-app")),
seen("qbit", "metamesh.transport/torrent@1", Some("http://qbit:3000"), None),
seen("other-ipfs", "metamesh.transport/ipfs@1", Some("http://other-ipfs:3000"), Some("metawatch-share")),
seen("no-endpoint", "metamesh.transport/ipfs@1", None, None),
];
let configured = vec![("nzb".to_string(), "http://metashare-nzb:3000".to_string())];
let r = classify("metamesh.transport/*", "metashare-app", 10, resources, &configured);
assert_eq!(r.summary.nodes, 10);
assert_eq!(r.summary.capable, 3, "a resource without an http endpoint is not a candidate");
assert_eq!((r.summary.configured, r.summary.addable, r.summary.bound_elsewhere), (1, 1, 1));
assert_eq!(r.candidates[0].instance, "qbit", "addable rows sort first");
assert_eq!(r.candidates[1].configured_as.as_deref(), Some("nzb"));
assert_eq!(r.candidates[2].state, CandidateState::BoundElsewhere);
let json = serde_json::to_value(&r).unwrap();
assert_eq!(json["self"], "metashare-app");
assert_eq!(json["summary"]["boundElsewhere"], 1);
assert_eq!(json["candidates"][2]["state"], "bound-elsewhere");
}
#[test]
fn suggested_names_are_list_safe() {
assert_eq!(suggested_name("MetaShare NZB"), "metashare-nzb");
assert_eq!(suggested_name(""), "plugin");
}
#[tokio::test]
async fn scan_reports_what_the_node_heard() {
use crate::beacon::{BeaconNode, Config, Message};
let port = std::net::UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
let mut cfg = Config::new("consumer").instance("me");
cfg.port = port;
let node = BeaconNode::spawn(cfg).unwrap();
let ad = Message::advertise(
NodeInfo { name: "meta-feeder-tmdb".into(), instance: "tmdb-feeder".into(), ..Default::default() },
vec![Resource::new("feeder", ["metamesh.feeder/tmdb"]).with_endpoint("http", "http://tmdb-feeder:8080")],
);
let s = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap();
s.send_to(&serde_json::to_vec(&ad).unwrap(), ("127.0.0.1", port)).await.unwrap();
let r = scan(&node, "metamesh.feeder/*", &[], Duration::from_millis(200)).await;
assert_eq!(r.summary.nodes, 1);
assert_eq!(r.summary.addable, 1);
assert_eq!(r.candidates[0].url, "http://tmdb-feeder:8080");
assert_eq!(r.candidates[0].suggested_name, "tmdb-feeder");
assert!(r.duration_ms >= 200);
}
}
+7 -7
View File
@@ -3,7 +3,7 @@
<head> <head>
<meta charset="utf-8" /> <meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<title>feeder config</title> <title>plugin config</title>
<style> <style>
:root { color-scheme: dark; } :root { color-scheme: dark; }
* { box-sizing: border-box; } * { box-sizing: border-box; }
@@ -47,7 +47,7 @@
</head> </head>
<body> <body>
<h1 id="title">Plugin configuration</h1> <h1 id="title">Plugin configuration</h1>
<p class="sub">Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked <span class="req">*</span> are required.</p> <p class="sub">Secrets are write-only — set values are shown as “set”, never echoed back. Saving restarts the plugin to apply. Fields marked <span class="req">*</span> are required.</p>
<div id="banner" class="banner"></div> <div id="banner" class="banner"></div>
<form id="form"></form> <form id="form"></form>
<div class="bar" id="bar" style="display:none"> <div class="bar" id="bar" style="display:none">
@@ -220,14 +220,14 @@ async function save() {
}); });
if (!res.ok) { if (!res.ok) {
const detail = (await res.text().catch(() => "")).trim(); const detail = (await res.text().catch(() => "")).trim();
throw new Error(detail ? `the feeder rejected it (HTTP ${res.status}): ${detail}` throw new Error(detail ? `the plugin rejected it (HTTP ${res.status}): ${detail}`
: `the feeder returned HTTP ${res.status}`); : `the plugin returned HTTP ${res.status}`);
} }
const j = await res.json().catch(() => ({})); const j = await res.json().catch(() => ({}));
if (j.reloading) { if (j.reloading) {
// The feeder is restarting itself (~2-3s) to apply the change. Poll it // The feeder is restarting itself (~2-3s) to apply the change. Poll it
// back up, then refresh the form so secret pills reflect stored state. // back up, then refresh the form so secret pills reflect stored state.
banner("warn", "Saved — reloading the feeder to apply (~3 s)…"); banner("warn", "Saved — reloading the plugin to apply (~3 s)…");
let tries = 6; let tries = 6;
const poll = async () => { const poll = async () => {
try { await load(); banner("ok", "Reloaded — new config applied."); } try { await load(); banner("ok", "Reloaded — new config applied."); }
@@ -235,7 +235,7 @@ async function save() {
}; };
setTimeout(poll, 3000); setTimeout(poll, 3000);
} else { } else {
if (j.restart_required) banner("warn", "Saved. Restart the feeder container for changes to take effect."); if (j.restart_required) banner("warn", "Saved. Restart the plugin container for changes to take effect.");
else banner("ok", "Saved."); else banner("ok", "Saved.");
// Re-fetch so secret pills reflect the new stored state. // Re-fetch so secret pills reflect the new stored state.
await load(); await load();
@@ -243,7 +243,7 @@ async function save() {
} catch (e) { } catch (e) {
const msg = (e && e.message) ? e.message : String(e); const msg = (e && e.message) ? e.message : String(e);
const friendly = /Failed to fetch|NetworkError|load failed/i.test(msg) const friendly = /Failed to fetch|NetworkError|load failed/i.test(msg)
? "couldn't reach the feeder — it may be restarting; wait a moment and try again." ? "couldn't reach the plugin — it may be restarting; wait a moment and try again."
: msg; : msg;
banner("err", "Could not save: " + friendly); banner("err", "Could not save: " + friendly);
} finally { } finally {
+290 -18
View File
@@ -206,6 +206,76 @@ pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option<Strin
Some(format!("b{}", base32_lower_no_padding(&wire))) Some(format!("b{}", base32_lower_no_padding(&wire)))
} }
/// Decode a self-describing `nzb-release` cid into `(api_base, id)` — the exact
/// inverse of [`compute_nzb_release_cid`], string-level (no `cid` crate).
///
/// `api_base` is scheme-less authority + optional path (`api.nzbgeek.info/api`);
/// match credentials on its authority (split at the first `/`), grab against
/// `https://{api_base}`. `id` is lowercase hex, re-encoded from the digest — so
/// an **odd-length** id comes back left-padded with `0` (the encoder pads it),
/// which Newznab `t=get` accepts as the same release.
///
/// `None` on a wrong codec, a non-identity multihash, or a malformed digest.
pub fn decode_nzb_release_cid(cid: &str) -> Option<(String, String)> {
let digest = identity_locator_digest(cid, NZB_RELEASE_CODEC)?;
let mut pos = 0usize;
let base_len = read_varint(&digest, &mut pos)? as usize;
let rest = digest.get(pos..)?;
if rest.len() <= base_len {
return None; // need the base AND at least one id byte
}
let (base, id) = rest.split_at(base_len);
let api_base = std::str::from_utf8(base).ok()?.to_string();
let id_hex: String = id.iter().map(|b| format!("{b:02x}")).collect();
Some((api_base, id_hex))
}
/// The multicodec of a base32 (`b…`) CIDv1, or `None` for anything else
/// (CIDv0 `Qm…`, other multibases, garbage). Dispatch on this — never on the
/// multihash, which is identity (`0x00`) for every locator family.
pub fn codec_of(cid: &str) -> Option<u64> {
let wire = base32_lower_decode(cid.strip_prefix('b')?)?;
let mut pos = 0usize;
if read_varint(&wire, &mut pos)? != 1 {
return None;
}
read_varint(&wire, &mut pos)
}
/// The digest of a well-formed identity-multihash CIDv1 carrying `codec`, or
/// `None`. Shared by the locator decoders.
fn identity_locator_digest(cid: &str, codec: u64) -> Option<Vec<u8>> {
let wire = base32_lower_decode(cid.strip_prefix('b')?)?;
let mut pos = 0usize;
if read_varint(&wire, &mut pos)? != 1
|| read_varint(&wire, &mut pos)? != codec
|| read_varint(&wire, &mut pos)? != 0x00
{
return None;
}
let len = read_varint(&wire, &mut pos)? as usize;
let digest = wire.get(pos..)?;
(digest.len() == len).then(|| digest.to_vec())
}
/// Unsigned LEB128 varint at `*pos`, advancing it.
fn read_varint(bytes: &[u8], pos: &mut usize) -> Option<u64> {
let mut value = 0u64;
let mut shift = 0u32;
loop {
let byte = *bytes.get(*pos)?;
*pos += 1;
value |= u64::from(byte & 0x7f) << shift;
if byte & 0x80 == 0 {
return Some(value);
}
shift += 7;
if shift >= 64 {
return None;
}
}
}
/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the /// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the
/// id is all-hex by construction; odd-length ids are left-padded with a `0` /// id is all-hex by construction; odd-length ids are left-padded with a `0`
/// nibble. Returns `None` on any non-hex byte (defensive). /// nibble. Returns `None` on any non-hex byte (defensive).
@@ -248,8 +318,12 @@ fn decode_hex_id(id: &str) -> Option<Vec<u8>> {
/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe. /// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe.
pub const CARD_LOCATOR_CODEC: u64 = 0x1007; pub const CARD_LOCATOR_CODEC: u64 = 0x1007;
/// Digest byte ceiling for a card locator, matching meta-share's /// Digest byte ceiling for the locators that still carry one — `yt-video`
/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. /// (via [`kinded_locator_cid`]) and `ext-play`. Matches meta-share's
/// `MAX_MULTIHASH_SIZE`, whose `CidGeneric<64>` rejects a longer multihash.
///
/// ⚠ **[`compute_card_cid`] is deliberately NOT subject to this** — see its
/// doc. The name is kept because both remaining users were written against it.
const CARD_LOCATOR_MAX_DIGEST: usize = 64; const CARD_LOCATOR_MAX_DIGEST: usize = 64;
/// Encode a self-describing **card locator** CID from a metadata source and /// Encode a self-describing **card locator** CID from a metadata source and
@@ -270,8 +344,28 @@ const CARD_LOCATOR_MAX_DIGEST: usize = 64;
/// operation driven by an explicit cross-source id mapping — never inferred /// operation driven by an explicit cross-source id mapping — never inferred
/// here (see `meta-gateway/docs/others/card-tier-search.md` §9). /// here (see `meta-gateway/docs/others/card-tier-search.md` §9).
/// ///
/// Returns `None` when `source` is empty or the digest would overflow the /// # No length ceiling, deliberately
/// multihash budget — the caller drops that card. ///
/// A CIDv1 digest length is a varint, so the wire format imposes no limit; the
/// 64-byte figure elsewhere in this file is Rust's fixed-size `CidGeneric<64>`,
/// which meta-share needs only because beetswap is
/// `Behaviour<const MAX_MULTIHASH_SIZE, B>`. A card is never bitswap content
/// (seeding is gated on sha2-256), so that ceiling does not apply — exactly the
/// reasoning `METADATA_KEYS.md` §2.1 already records for the `url` locator,
/// which ships URLs past it today. meta-core's Go `Decode` hand-parses varints
/// with no cap and reads these correctly.
///
/// This is load-bearing for URL-shaped ids: a card keyed on a work's own
/// address (`("weebcentral.com", "/series/…/some-long-slug")`) runs to ~88
/// bytes, and capping would silently drop the card rather than mint it.
///
/// ⚠ The one consumer that still cares is meta-share's `cid_rank`, which parses
/// through `MsCid::from_str`. A locator it cannot parse scores `RANK_UNKNOWN`
/// instead of `RANK_LOCATOR`; harmless for a card (its record carries one CID,
/// so it wins the canonical election at any rank) but pinned by a golden vector
/// in `cid-rank-vectors.json` so the two implementations cannot drift unseen.
///
/// Returns `None` only when `source` or `id` is empty.
pub fn compute_card_cid(source: &str, id: &str) -> Option<String> { pub fn compute_card_cid(source: &str, id: &str) -> Option<String> {
if source.is_empty() || id.is_empty() { if source.is_empty() || id.is_empty() {
return None; return None;
@@ -284,9 +378,6 @@ pub fn compute_card_cid(source: &str, id: &str) -> Option<String> {
write_pb_varint(source_b.len() as u64, &mut digest); write_pb_varint(source_b.len() as u64, &mut digest);
digest.extend_from_slice(source_b); digest.extend_from_slice(source_b);
digest.extend_from_slice(id_b); digest.extend_from_slice(id_b);
if digest.len() > CARD_LOCATOR_MAX_DIGEST {
return None;
}
// CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest]
let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
@@ -378,8 +469,39 @@ pub fn compute_ext_play_cid(url: &str) -> Option<String> {
Some(identity_locator_cid(EXT_PLAY_CODEC, digest)) Some(identity_locator_cid(EXT_PLAY_CODEC, digest))
} }
/// Custom multicodec for a fetchable **`provider-file`** locator —
/// `docs/cid-formats.md` §8. One codec for every provider whose bytes are
/// fetched the same way; the provider lives inside the digest:
/// `varint(len(source)) ‖ source ‖ id`. Framing byte-identical to `card`
/// (`0x1007`) — only the codec differs — but the byte path *redeems* it (through
/// the claiming feeder's `/compute`) instead of 404ing.
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
/// Encode a `provider-file` CID from a registered source token
/// (`opensubtitles`) and that source's id (`file:7061834`). `None` on an empty
/// part or a digest past 64 bytes — the caller drops that row rather than emit
/// an address no peer can parse.
pub fn compute_provider_file_cid(source: &str, id: &str) -> Option<String> {
kinded_locator_cid(PROVIDER_FILE_CODEC, source, id)
}
/// Inverse of [`compute_provider_file_cid`]: `(source, id)`. Matches the
/// **codec slot** — a `card` with identical framing does not decode here.
pub fn decode_provider_file_cid(cid: &str) -> Option<(String, String)> {
let digest = identity_locator_digest(cid, PROVIDER_FILE_CODEC)?;
let mut pos = 0usize;
let source_len = read_varint(&digest, &mut pos)? as usize;
let source = digest.get(pos..pos.checked_add(source_len)?)?;
let id = digest.get(pos + source_len..)?;
Some((
String::from_utf8(source.to_vec()).ok()?,
String::from_utf8(id.to_vec()).ok()?,
))
}
/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used /// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used
/// by [`compute_card_cid`] and [`compute_yt_video_cid`]. /// by [`compute_card_cid`], [`compute_yt_video_cid`] and
/// [`compute_provider_file_cid`].
fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option<String> { fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option<String> {
if kind.is_empty() || id.is_empty() { if kind.is_empty() || id.is_empty() {
return None; return None;
@@ -1067,19 +1189,88 @@ mod tests {
assert_eq!(&digest[5..], b"tv:95479"); assert_eq!(&digest[5..], b"tv:95479");
} }
/// Empty inputs are rejected rather than encoded as an ambiguous locator, /// Empty inputs are rejected rather than encoded as an ambiguous locator.
/// and an id past the 64-byte multihash budget (meta-share's
/// `CidGeneric<64>`) returns `None` so the caller drops the card instead of
/// emitting a cid that peer would refuse to parse.
#[test] #[test]
fn card_cid_rejects_empty_and_oversized() { fn card_cid_rejects_empty_halves() {
assert!(compute_card_cid("", "tv:1").is_none()); assert!(compute_card_cid("", "tv:1").is_none());
assert!(compute_card_cid("tmdb", "").is_none()); assert!(compute_card_cid("tmdb", "").is_none());
let huge = "x".repeat(CARD_LOCATOR_MAX_DIGEST); }
assert!(compute_card_cid("tmdb", &huge).is_none());
// Exactly at the ceiling still encodes. /// ⚠ A card locator has **no length ceiling**, and this is the test that
let fits = "x".repeat(CARD_LOCATOR_MAX_DIGEST - 5); // varint(4) + "tmdb" /// says so on purpose — it used to assert the opposite.
assert!(compute_card_cid("tmdb", &fits).is_some()); ///
/// The 64-byte figure is meta-share's `CidGeneric<64>` bitswap alias, and a
/// card is never bitswap content. Capping here would silently drop exactly
/// the cards a URL-shaped identity exists to mint: a `weebcentral.com` work
/// path digests to ~88 bytes. Same reasoning `METADATA_KEYS.md` §2.1
/// already records for the `url` locator.
#[test]
fn card_cid_has_no_length_ceiling() {
let long_id = "/series/01J76XYGF1RKSPWA4TQ6GGSVFV/\
Ive-Been-Killing-Slimes-for-300-Years-The-Red-Dragon-Academy-for-Girls";
let cid = compute_card_cid("weebcentral.com", long_id).expect("no ceiling");
let (source, id) = split_card_digest(&cid);
assert!(
source.len() + id.len() > CARD_LOCATOR_MAX_DIGEST,
"fixture must actually exceed the old cap, got {}",
source.len() + id.len()
);
assert_eq!(source, "weebcentral.com");
assert_eq!(id, long_id);
}
/// The host-form identity: the source slot names the site that *owns* the
/// work, never the tool that reached it. Pinned as a literal because
/// `meta-read`'s `card_cid::encode` carries the mirrored assertion — the two
/// addressing one work differently is the failure the whole scheme cannot
/// survive.
#[test]
fn host_form_card_cid_golden_vector() {
let cid = compute_card_cid("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512")
.expect("card cid");
assert_eq!(
cid,
"bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
);
}
/// `varint(source_len) ‖ source ‖ id` back out of a card locator, so the
/// tests above can assert on the halves without a decoder in this crate.
fn split_card_digest(cid: &str) -> (String, String) {
let raw = base32_lower_decode(&cid[1..]);
let mut i = 0usize;
let mut rd = || {
let (mut n, mut sh) = (0u64, 0u32);
loop {
let b = raw[i];
i += 1;
n |= u64::from(b & 0x7f) << sh;
if b & 0x80 == 0 {
return n;
}
sh += 7;
}
};
assert_eq!(rd(), 1, "CIDv1");
assert_eq!(rd(), CARD_LOCATOR_CODEC);
assert_eq!(rd(), 0, "identity multihash");
let len = rd() as usize;
let digest = &raw[i..i + len];
let mut j = 0usize;
let (mut sl, mut sh) = (0usize, 0u32);
loop {
let b = digest[j];
j += 1;
sl |= usize::from(b & 0x7f) << sh;
if b & 0x80 == 0 {
break;
}
sh += 7;
}
(
String::from_utf8(digest[j..j + sl].to_vec()).unwrap(),
String::from_utf8(digest[j + sl..].to_vec()).unwrap(),
)
} }
// ---- nzb-posting (0x1003) — the dedup contract ----------------------- // ---- nzb-posting (0x1003) — the dedup contract -----------------------
@@ -1299,3 +1490,84 @@ mod roundtrip_tests {
assert_eq!(decode_url_cid(&card), None); assert_eq!(decode_url_cid(&card), None);
} }
} }
#[cfg(test)]
mod redeem_locator_tests {
use super::*;
/// `docs/cid-formats.md` §8.2 golden vector — moved here from
/// meta-feeder-opensubtitles' local `locator.rs`; the vector proves the
/// move changed no bytes.
#[test]
fn provider_file_matches_the_documented_vector() {
assert_eq!(
compute_provider_file_cid("opensubtitles", "file:7061834").as_deref(),
Some("bagfcaaa2bvxxazloon2we5djorwgk43gnfwgkorxga3dcobtgq")
);
}
#[test]
fn provider_file_round_trips_and_refuses_other_codecs() {
let cid = compute_provider_file_cid("opensubtitles", "file:12658754").unwrap();
assert_eq!(
decode_provider_file_cid(&cid),
Some(("opensubtitles".to_string(), "file:12658754".to_string()))
);
// Identical framing, different codec: a card must not decode as a file.
let card = compute_card_cid("tmdb", "tv:95479").unwrap();
assert_eq!(card, "bagdsaaanar2g2zdcor3duojvgq3ts");
assert_eq!(decode_provider_file_cid(&card), None);
assert_eq!(decode_provider_file_cid("not-a-cid"), None);
assert_eq!(decode_provider_file_cid(""), None);
}
#[test]
fn provider_file_digest_ceiling_is_64_bytes() {
// varint(13) + "opensubtitles" = 14 bytes, so the id may use 50.
assert!(compute_provider_file_cid("opensubtitles", &"x".repeat(50)).is_some());
assert!(compute_provider_file_cid("opensubtitles", &"x".repeat(51)).is_none());
}
#[test]
fn nzb_release_round_trips() {
let cid = compute_nzb_release_cid("api.nzbgeek.info/api", "0123abcdef").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("api.nzbgeek.info/api".to_string(), "0123abcdef".to_string()))
);
// A host:port authority survives too.
let cid = compute_nzb_release_cid("127.0.0.1:8080", "ff00").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("127.0.0.1:8080".to_string(), "ff00".to_string()))
);
}
/// The encoder left-pads an odd-length id, so the decode is the padded form.
#[test]
fn nzb_release_odd_length_id_comes_back_padded() {
let cid = compute_nzb_release_cid("api.nzb.life", "abc").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("api.nzb.life".to_string(), "0abc".to_string()))
);
}
#[test]
fn nzb_release_refuses_other_codecs() {
let card = compute_card_cid("tmdb", "tv:95479").unwrap();
assert_eq!(decode_nzb_release_cid(&card), None);
assert_eq!(decode_nzb_release_cid("bafkreigarbage"), None);
}
#[test]
fn codec_of_reads_the_codec_slot() {
let nzb = compute_nzb_release_cid("api.nzb.life", "ab").unwrap();
assert_eq!(codec_of(&nzb), Some(NZB_RELEASE_CODEC));
let file = compute_provider_file_cid("opensubtitles", "file:1").unwrap();
assert_eq!(codec_of(&file), Some(PROVIDER_FILE_CODEC));
assert_eq!(codec_of(&compute_ipfs_cid(b"hello")), Some(0x55));
assert_eq!(codec_of("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), None);
assert_eq!(codec_of(""), None);
}
}
+40
View File
@@ -8,6 +8,14 @@
//! hashing-into-the-blockstore, and the meta-core store-back to itself. //! hashing-into-the-blockstore, and the meta-core store-back to itself.
//! //!
//! Deliberately libp2p-free and blockstore-free — see this crate's `Cargo.toml`. //! Deliberately libp2p-free and blockstore-free — see this crate's `Cargo.toml`.
//!
//! With the `transport` feature it also carries the **transport-plugin**
//! contract ([`transport`]): the same split, applied to meta-share — a
//! protocol-agnostic hull and one sidecar per byte-moving protocol.
//!
//! [`beacon`] is beacon v2 local discovery. Both harnesses advertise through it;
//! with `default-features = false, features = ["beacon"]` it is all a meta-*
//! service pulls in.
// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it // `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it
// directly rather than boxing on every embedded-DB call. Boxing each would be // directly rather than boxing on every embedded-DB call. Boxing each would be
@@ -21,34 +29,66 @@
/// `metacore-app` service the AppStore feeder apps ship on the shared network. /// `metacore-app` service the AppStore feeder apps ship on the shared network.
pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000"; pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000";
#[cfg(feature = "beacon")]
pub mod beacon;
#[cfg(feature = "feeder")]
pub mod budget; pub mod budget;
#[cfg(feature = "feeder")]
pub mod cache; pub mod cache;
#[cfg(feature = "feeder")]
pub mod common; pub mod common;
#[cfg(feature = "feeder")]
pub mod config; pub mod config;
#[cfg(feature = "feeder")]
pub mod domain; pub mod domain;
#[cfg(feature = "feeder")]
pub mod enrich; pub mod enrich;
#[cfg(feature = "filename")]
pub mod filename_meta; pub mod filename_meta;
#[cfg(feature = "feeder")]
pub mod hash; pub mod hash;
#[cfg(feature = "feeder")]
pub mod lang; pub mod lang;
#[cfg(feature = "feeder")]
pub mod licence; pub mod licence;
#[cfg(feature = "feeder")]
pub mod meta_core; pub mod meta_core;
#[cfg(feature = "feeder")]
pub mod plugin; pub mod plugin;
#[cfg(feature = "feeder")]
pub mod query; pub mod query;
#[cfg(feature = "feeder")]
pub mod query_eval; pub mod query_eval;
#[cfg(feature = "feeder")]
pub mod serve; pub mod serve;
#[cfg(feature = "transport")]
pub mod transport;
#[cfg(feature = "feeder")]
pub mod types; pub mod types;
#[cfg(feature = "feeder")]
pub use budget::RateBudget; pub use budget::RateBudget;
#[cfg(feature = "feeder")]
pub use config::{ConfigField, ConfigSchema, FieldKind}; pub use config::{ConfigField, ConfigSchema, FieldKind};
#[cfg(feature = "feeder")]
pub use licence::licence_from_url; pub use licence::licence_from_url;
#[cfg(feature = "feeder")]
pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher}; pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher};
#[cfg(feature = "feeder")]
pub use meta_core::FeederStore; pub use meta_core::FeederStore;
#[cfg(feature = "feeder")]
pub use plugin::{ pub use plugin::{
upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry,
RedeemClaim,
}; };
#[cfg(feature = "feeder")]
pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter};
#[cfg(feature = "feeder")]
pub use serve::{ pub use serve::{
configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto,
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
RedeemsResponse,
}; };
#[cfg(feature = "feeder")]
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
+94
View File
@@ -17,6 +17,7 @@ use std::path::Path;
use async_trait::async_trait; use async_trait::async_trait;
use futures::stream::BoxStream; use futures::stream::BoxStream;
use serde::{Deserialize, Serialize};
// Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the // Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the
// gateway's original `plugin.rs` re-exported it the same way). // gateway's original `plugin.rs` re-exported it the same way).
@@ -95,6 +96,66 @@ pub enum HashKind {
YtVideo, YtVideo,
} }
/// `RedeemClaim::codec` for a Newznab release locator (`0x1005`).
pub const REDEEM_CODEC_NZB_RELEASE: &str = "nzb-release";
/// `RedeemClaim::codec` for a `provider-file` locator (`0x100A`).
pub const REDEEM_CODEC_PROVIDER_FILE: &str = "provider-file";
/// `RedeemClaim::field` naming the `.nzb` manifest pointer on a release record.
pub const POINTER_FIELD_MANIFEST: &str = "manifest";
/// `RedeemClaim::field` naming the fetched-file pointer on a locator record.
pub const POINTER_FIELD_FILE: &str = "file";
/// A plugin's claim that it can **redeem** a locator family: turn a locator cid
/// into the bytes it names, through `POST /compute` with `record_id` = the
/// locator cid.
///
/// Redeeming usually spends a metered upstream quota (a Newznab `t=get` grab,
/// an OpenSubtitles `/download`), so the gateway routes a redeem only to the
/// plugins claiming the cid's codec, one at a time, and only on a real play. It
/// stores the returned bytes and writes their cid onto the locator record under
/// [`field`](Self::field), so the quota is spent once per file.
///
/// Served at `GET /manifest` (per plugin) and `GET /redeems`. Reflects the
/// plugin's *current* config: a plugin with no credential for a family claims
/// nothing for it.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct RedeemClaim {
/// Locator family: [`REDEEM_CODEC_NZB_RELEASE`] or [`REDEEM_CODEC_PROVIDER_FILE`].
pub codec: String,
/// Pointer field the gateway writes the redeemed bytes' cid under:
/// [`POINTER_FIELD_MANIFEST`] or [`POINTER_FIELD_FILE`].
pub field: String,
/// Indexer hosts this plugin holds a key for (`nzb-release`) — bare
/// authority, lowercase, never the key itself.
#[serde(default)]
pub hosts: Vec<String>,
/// Provider sources this plugin redeems (`provider-file`), e.g. `opensubtitles`.
#[serde(default)]
pub sources: Vec<String>,
}
impl RedeemClaim {
/// Claim `nzb-release` locators for these indexer hosts.
pub fn nzb_release(hosts: Vec<String>) -> Self {
Self {
codec: REDEEM_CODEC_NZB_RELEASE.to_string(),
field: POINTER_FIELD_MANIFEST.to_string(),
hosts,
sources: Vec::new(),
}
}
/// Claim `provider-file` locators for these sources.
pub fn provider_file(sources: Vec<String>) -> Self {
Self {
codec: REDEEM_CODEC_PROVIDER_FILE.to_string(),
field: POINTER_FIELD_FILE.to_string(),
hosts: Vec::new(),
sources,
}
}
}
/// Startup-only configuration error returned by [`FeederPlugin::configure`]. /// Startup-only configuration error returned by [`FeederPlugin::configure`].
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum ConfigError { pub enum ConfigError {
@@ -215,6 +276,20 @@ pub trait FeederPlugin: Send + Sync + 'static {
&[] &[]
} }
/// Folder under meta-core's `/files/plugin/` where the gateway stores bytes
/// this plugin redeems (e.g. `meta-feeder-usenet`). Stable across
/// deployments — unlike the feeder's compose service name. Default: none.
fn package(&self) -> Option<&'static str> {
None
}
/// Locator families this plugin can redeem through
/// [`compute_outcomes`](FeederPlugin::compute_outcomes) — see
/// [`RedeemClaim`]. Computed from the current config. Default: none.
fn redeems(&self) -> Vec<RedeemClaim> {
Vec::new()
}
/// Self-describing config schema. The feeder SDK serves this at /// Self-describing config schema. The feeder SDK serves this at
/// `GET /config/schema` and renders a generic form from it — the gateway and /// `GET /config/schema` and renders a generic form from it — the gateway and
/// UI carry no per-plugin field knowledge. Default: no configuration. /// UI carry no per-plugin field knowledge. Default: no configuration.
@@ -255,4 +330,23 @@ mod tests {
assert!(msg.contains("scihub")); assert!(msg.contains("scihub"));
assert!(msg.contains("mirrors")); assert!(msg.contains("mirrors"));
} }
/// The JSON keys are the gateway/meta-share contract — pin them.
#[test]
fn redeem_claim_wire_shape() {
let claim = RedeemClaim::nzb_release(vec!["api.nzb.life".into()]);
assert_eq!(
serde_json::to_value(&claim).unwrap(),
serde_json::json!({
"codec": "nzb-release",
"field": "manifest",
"hosts": ["api.nzb.life"],
"sources": []
})
);
// hosts/sources may be omitted on the wire.
let parsed: RedeemClaim =
serde_json::from_str(r#"{"codec":"provider-file","field":"file"}"#).unwrap();
assert_eq!(parsed, RedeemClaim::provider_file(vec![]));
}
} }
+218 -7
View File
@@ -10,6 +10,17 @@
//! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`] //! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`]
//! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404) //! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404)
//! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404) //! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404)
//! - `GET /redeems` → [`RedeemsResponse`] (current redeem claims)
//!
//! ## Redeeming a locator
//!
//! A plugin that advertises a [`RedeemClaim`] answers `POST /compute` with
//! `record_id` = a locator cid of that family (a Newznab `nzb-release`, an
//! OpenSubtitles `provider-file`) by fetching the bytes it names — usually
//! spending a metered quota — and returning ONE `Sha2_256` outcome carrying
//! them. `NotFound` means "not mine" (e.g. no key for that indexer host), so the
//! gateway tries the next claimer. The claims are re-read from `GET /redeems`
//! so a config change shows up without restarting the gateway.
//! //!
//! ## Bytes & the v1 simplification //! ## Bytes & the v1 simplification
//! //!
@@ -21,7 +32,7 @@
//! documented follow-up — those outcomes are metadata-only anyway //! documented follow-up — those outcomes are metadata-only anyway
//! (`bytes: None`), so they never ride this path. //! (`bytes: None`), so they never ride this path.
use std::collections::HashMap; use std::collections::{BTreeMap, HashMap};
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::{Path as FsPath, PathBuf}; use std::path::{Path as FsPath, PathBuf};
use std::sync::Arc; use std::sync::Arc;
@@ -41,7 +52,7 @@ use serde_json::Value;
use tracing::{info, warn}; use tracing::{info, warn};
use crate::config::{self, CONFIG_PAGE_HTML}; use crate::config::{self, CONFIG_PAGE_HTML};
use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome, RedeemClaim};
use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError}; use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError};
use crate::types::{DiscoveryRecord, GatewayError, PluginHealth}; use crate::types::{DiscoveryRecord, GatewayError, PluginHealth};
@@ -94,6 +105,22 @@ pub struct PluginManifest {
pub id: String, pub id: String,
pub served_file_types: Vec<String>, pub served_file_types: Vec<String>,
pub served_content_kinds: Vec<String>, pub served_content_kinds: Vec<String>,
/// [`FeederPlugin::package`] — the `/files/plugin/<package>/` folder redeemed
/// bytes are stored under. `null` when the plugin redeems nothing.
#[serde(default)]
pub package: Option<String>,
/// [`FeederPlugin::redeems`] at the time the manifest was served.
#[serde(default)]
pub redeems: Vec<RedeemClaim>,
}
/// `GET /redeems` — every hosted plugin's current [`RedeemClaim`]s, keyed by
/// `upstream_id` (an empty list for a plugin that claims nothing). Polled by
/// the gateway, so a credential added on the config page is advertised without
/// a gateway restart.
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct RedeemsResponse {
pub redeems: BTreeMap<String, Vec<RedeemClaim>>,
} }
/// `GET /manifest` — the feeder's hosted plugins + their served types. The /// `GET /manifest` — the feeder's hosted plugins + their served types. The
@@ -224,6 +251,17 @@ impl AppState {
/// status semantics the gateway's `RemoteFeederPlugin` will translate back /// status semantics the gateway's `RemoteFeederPlugin` will translate back
/// into a `GatewayError` on the core side. /// into a `GatewayError` on the core side.
fn gateway_error_response(e: GatewayError) -> Response { fn gateway_error_response(e: GatewayError) -> Response {
// A 429 carries the wait as a `Retry-After` header too, so the gateway can
// forward it to the client (a redeem blocked on a spent quota) instead of
// parsing the message.
if let GatewayError::RateLimited { retry_after_s } = &e {
return (
StatusCode::TOO_MANY_REQUESTS,
[(axum::http::header::RETRY_AFTER, retry_after_s.to_string())],
format!("rate limited; retry in {retry_after_s}s"),
)
.into_response();
}
let (status, msg) = match e { let (status, msg) = match e {
GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()), GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()),
GatewayError::RateLimited { retry_after_s } => ( GatewayError::RateLimited { retry_after_s } => (
@@ -240,8 +278,15 @@ fn gateway_error_response(e: GatewayError) -> Response {
// -------- handlers --------------------------------------------------------- // -------- handlers ---------------------------------------------------------
async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> { async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
let mut plugins: Vec<PluginManifest> = state Json(build_manifest(&state.plugins, &state.version))
.plugins }
/// The `/manifest` body — also what the beacon advertise's `rev` is a hash of.
fn build_manifest(
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
version: &str,
) -> ManifestResponse {
let mut plugins: Vec<PluginManifest> = plugins
.values() .values()
.map(|p| PluginManifest { .map(|p| PluginManifest {
id: p.upstream_id().to_string(), id: p.upstream_id().to_string(),
@@ -251,12 +296,41 @@ async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
.iter() .iter()
.map(|s| s.to_string()) .map(|s| s.to_string())
.collect(), .collect(),
package: p.package().map(str::to_string),
redeems: p.redeems(),
}) })
.collect(); .collect();
plugins.sort_by(|a, b| a.id.cmp(&b.id)); plugins.sort_by(|a, b| a.id.cmp(&b.id));
Json(ManifestResponse { ManifestResponse {
feeder_version: state.version.clone(), feeder_version: version.to_string(),
plugins, plugins,
}
}
/// The feeder's beacon v2 resource: one `metamesh.feeder/<upstream_id>` cap per
/// hosted plugin, the manifest endpoint, and `rev` = hash of the manifest.
fn beacon_resource(
plugins: &HashMap<String, Arc<dyn FeederPlugin>>,
version: &str,
listen_port: u16,
) -> crate::beacon::Resource {
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
let m = build_manifest(plugins, version);
let mut r = Resource::new("feeder", m.plugins.iter().map(|p| caps::feeder(&p.id)))
.with_endpoint("http", advertise_url(listen_port))
.with_endpoint("manifest", "/manifest");
r.rev = Some(rev_of(&m));
r.binds = binds_from_env();
r
}
async fn redeems(State(state): State<AppState>) -> Json<RedeemsResponse> {
Json(RedeemsResponse {
redeems: state
.plugins
.values()
.map(|p| (p.upstream_id().to_string(), p.redeems()))
.collect(),
}) })
} }
@@ -492,6 +566,7 @@ pub fn router(
.route("/compute", post(compute)) .route("/compute", post(compute))
.route("/fetch/:upstream_id/:record_id", get(fetch)) .route("/fetch/:upstream_id/:record_id", get(fetch))
.route("/blob/:upstream_id/:cid", get(blob)) .route("/blob/:upstream_id/:cid", get(blob))
.route("/redeems", get(redeems))
.route("/config", get(config_page)) .route("/config", get(config_page))
.route("/config/schema", get(config_schema)) .route("/config/schema", get(config_schema))
.route("/config/values", get(config_values_get).put(config_values_put)) .route("/config/values", get(config_values_get).put(config_values_put))
@@ -517,7 +592,8 @@ pub async fn serve_feeders(
let state_dir = state_dir.into(); let state_dir = state_dir.into();
let configured = configure_plugins(plugins, &state_dir)?; let configured = configure_plugins(plugins, &state_dir)?;
let version = env!("CARGO_PKG_VERSION").to_string(); let version = env!("CARGO_PKG_VERSION").to_string();
let app = router(configured, version, state_dir.clone()); let advertised = configured.clone();
let app = router(configured, version.clone(), state_dir.clone());
let listener = tokio::net::TcpListener::bind(listen).await?; let listener = tokio::net::TcpListener::bind(listen).await?;
info!( info!(
@@ -525,6 +601,18 @@ pub async fn serve_feeders(
%listen, %listen,
"feeder listening" "feeder listening"
); );
// Beacon v2: advertise once the port is bound, so a consumer that hears us
// can connect. The node lives for the process.
let port = listen.port();
let name = std::env::var("SERVICE_NAME").ok().filter(|s| !s.is_empty()).unwrap_or_else(|| {
let mut ids: Vec<&String> = advertised.keys().collect();
ids.sort();
ids.first().map_or_else(|| "meta-feeder".to_string(), |id| format!("meta-feeder-{id}"))
});
let beacon_version = version.clone();
let _beacon = crate::beacon::advertise_plugin(&name, &version, move || {
beacon_resource(&advertised, &beacon_version, port)
});
axum::serve(listener, app) axum::serve(listener, app)
.await .await
.map_err(anyhow::Error::from) .map_err(anyhow::Error::from)
@@ -541,6 +629,9 @@ struct DegradedPlugin {
reason: String, reason: String,
file_types: &'static [&'static str], file_types: &'static [&'static str],
content_kinds: &'static [&'static str], content_kinds: &'static [&'static str],
/// Preserved so the manifest keeps naming the plugin's folder. A degraded
/// plugin claims no redeems (it cannot redeem anything).
package: Option<&'static str>,
/// Preserved from the wrapped plugin so a degraded/unconfigured feeder still /// Preserved from the wrapped plugin so a degraded/unconfigured feeder still
/// serves its config form — otherwise the dashboard shows "no configuration /// serves its config form — otherwise the dashboard shows "no configuration
/// required" and the operator can never fix the very config that's missing. /// required" and the operator can never fix the very config that's missing.
@@ -586,6 +677,10 @@ impl FeederPlugin for DegradedPlugin {
self.content_kinds self.content_kinds
} }
fn package(&self) -> Option<&'static str> {
self.package
}
fn config_schema(&self) -> crate::config::ConfigSchema { fn config_schema(&self) -> crate::config::ConfigSchema {
self.schema.clone() self.schema.clone()
} }
@@ -608,6 +703,7 @@ pub fn configure_plugins(
// borrow across `configure(&mut self)` / the move into `out`. // borrow across `configure(&mut self)` / the move into `out`.
let file_types = plugin.served_file_types(); let file_types = plugin.served_file_types();
let content_kinds = plugin.served_content_kinds(); let content_kinds = plugin.served_content_kinds();
let package = plugin.package();
// Preserve the config surface so a degraded stand-in still serves the // Preserve the config surface so a degraded stand-in still serves the
// real config form (see DegradedPlugin.schema). // real config form (see DegradedPlugin.schema).
let schema = plugin.config_schema(); let schema = plugin.config_schema();
@@ -627,6 +723,7 @@ pub fn configure_plugins(
reason, reason,
file_types, file_types,
content_kinds, content_kinds,
package,
schema: schema.clone(), schema: schema.clone(),
}) })
}; };
@@ -759,4 +856,118 @@ mod tests {
assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. })); assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. }));
assert!(matches!(events[2], GatewaySearchEvent::Done)); assert!(matches!(events[2], GatewaySearchEvent::Done));
} }
/// A plugin that redeems `nzb-release` locators for one host.
struct StubRedeemer;
#[async_trait]
impl FeederPlugin for StubRedeemer {
fn upstream_id(&self) -> &'static str {
"redeemer"
}
fn configure(&mut self, _cache_dir: &FsPath) -> Result<(), ConfigError> {
Ok(())
}
async fn handle_query(
&self,
_q: &GatewayQuery,
_n: usize,
) -> Result<Vec<DiscoveryRecord>, GatewayError> {
Ok(vec![])
}
async fn compute_outcomes(
&self,
_record_id: &str,
) -> Result<Vec<HashOutcome>, GatewayError> {
Ok(vec![])
}
fn package(&self) -> Option<&'static str> {
Some("meta-feeder-stub")
}
fn redeems(&self) -> Vec<RedeemClaim> {
vec![RedeemClaim::nzb_release(vec!["api.example".into()])]
}
}
async fn serve_stubs() -> String {
let mut plugins: HashMap<String, Arc<dyn FeederPlugin>> = HashMap::new();
plugins.insert("redeemer".to_string(), Arc::new(StubRedeemer));
plugins.insert("stub".to_string(), Arc::new(StubStreaming));
let app = router(plugins, "test".to_string(), std::env::temp_dir());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{addr}")
}
/// The manifest carries `package` + `redeems` per plugin; a plugin that
/// claims nothing serializes `null` + `[]`.
#[tokio::test]
async fn manifest_carries_package_and_redeems() {
let base = serve_stubs().await;
let m: serde_json::Value = reqwest::get(format!("{base}/manifest"))
.await
.unwrap()
.json()
.await
.unwrap();
let redeemer = &m["plugins"][0];
assert_eq!(redeemer["id"], "redeemer");
assert_eq!(redeemer["package"], "meta-feeder-stub");
assert_eq!(
redeemer["redeems"],
serde_json::json!([{
"codec": "nzb-release", "field": "manifest",
"hosts": ["api.example"], "sources": []
}])
);
let stub = &m["plugins"][1];
assert_eq!(stub["package"], serde_json::Value::Null);
assert_eq!(stub["redeems"], serde_json::json!([]));
}
#[tokio::test]
async fn redeems_route_keys_claims_by_upstream_id() {
let base = serve_stubs().await;
let r: RedeemsResponse = reqwest::get(format!("{base}/redeems"))
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(
r.redeems.get("redeemer"),
Some(&vec![RedeemClaim::nzb_release(vec!["api.example".into()])])
);
assert_eq!(r.redeems.get("stub"), Some(&vec![]));
}
/// The gateway forwards a feeder 429's `Retry-After` — pin that it is set.
#[test]
fn rate_limited_sets_retry_after() {
let resp = gateway_error_response(GatewayError::RateLimited { retry_after_s: 3600 });
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
resp.headers()
.get(axum::http::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok()),
Some("3600")
);
// Other errors carry no Retry-After.
let resp = gateway_error_response(GatewayError::NotFound);
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
assert!(resp.headers().get(axum::http::header::RETRY_AFTER).is_none());
}
/// A manifest from a pre-1.3 feeder (no `package` / `redeems`) still parses.
#[test]
fn old_manifest_without_redeem_fields_parses() {
let m: PluginManifest = serde_json::from_str(
r#"{"id":"x","served_file_types":[],"served_content_kinds":[]}"#,
)
.unwrap();
assert!(m.package.is_none() && m.redeems.is_empty());
}
} }
+368
View File
@@ -0,0 +1,368 @@
//! CID parsing and the CID-*shape* knowledge every transport process shares.
//!
//! Not protocol logic — these are pure decoders over the multihash standard,
//! which is the one thing meta-share's hull stays tied to. Moved verbatim from
//! meta-share (`blockstore.rs` for the parse, `bt.rs` for the torrent shapes) so
//! the hull and the plugins classify a cid identically: a drift here would route
//! a torrent cid to the IPFS walker or vice versa.
use std::str::FromStr;
use anyhow::{Context, Result};
use cid::CidGeneric;
/// Multihash byte ceiling. Must match the gateway and meta-search side — it
/// feeds beetswap's `Behaviour<const MAX_MULTIHASH_SIZE, B>` in the ipfs plugin.
pub const MAX_MULTIHASH_SIZE: usize = 64;
/// The CID type every MetaMesh transport speaks.
pub type MsCid = CidGeneric<MAX_MULTIHASH_SIZE>;
/// Parse a `Record.cid` into an [`MsCid`].
///
/// **Strict**: a bare CIDv1 multibase string and nothing else, matching
/// `cid.Decode` in meta-core's `internal/cid/rank.go`. The `<algo>:<cid>` token
/// form and CIDv0 (`Qm…`) are both rejected — see meta-share's
/// `blockstore::parse_record_cid` history for why each one bit.
pub fn parse_record_cid(cid_str: &str) -> Result<MsCid> {
if cid_str.contains(':') {
anyhow::bail!(
"parse record cid `{cid_str}`: the `<algo>:<cid>` token form was removed; \
pass the bare CIDv1 (the multicodec already names the algorithm)"
);
}
let parsed =
MsCid::from_str(cid_str).with_context(|| format!("parse record cid `{cid_str}`"))?;
if parsed.version() != cid::Version::V1 {
anyhow::bail!(
"parse record cid `{cid_str}`: only CIDv1 is supported, got {:?}",
parsed.version()
);
}
Ok(parsed)
}
// ---------------------------------------------------------------------------
// BitTorrent cid shapes
// ---------------------------------------------------------------------------
/// Multihash code for sha1 — a v1 infohash is the sha1 of the info dict.
pub const MH_SHA1: u64 = 0x11;
const BT_V1_INFOHASH_LEN: usize = 20;
const BT_V2_INFOHASH_LEN: usize = 32;
/// `btih-v1-file`: one file inside a v1 torrent; digest `<infohash> ‖ varint(index)`.
pub const BTIH_V1_FILE_CODEC: u64 = 0x1001;
/// `btih-v2-file`: recognised so the error is precise, not yet fetchable.
pub const BTIH_V2_FILE_CODEC: u64 = 0x1002;
/// Is this a whole-torrent v1 infohash cid (sha1 multihash, 20 bytes)? The raw
/// codec `0x55` is shared with IPFS raw leaves, so key off the multihash.
pub fn is_btih(mscid: &MsCid) -> bool {
let mh = mscid.hash();
mh.code() == MH_SHA1 && mh.digest().len() == BT_V1_INFOHASH_LEN
}
/// The raw 20-byte v1 infohash of a `btih:` cid. Errors (a `400` upstream) for
/// anything that isn't sha1/20.
pub fn infohash20_from_mscid(mscid: &MsCid) -> Result<[u8; 20]> {
let mh = mscid.hash();
anyhow::ensure!(
mh.code() == MH_SHA1,
"not a v1 BitTorrent infohash CID (multihash code 0x{:x}, want 0x11/sha1)",
mh.code()
);
<[u8; 20]>::try_from(mh.digest())
.map_err(|_| anyhow::anyhow!("infohash digest is {} bytes, want 20", mh.digest().len()))
}
/// A decoded per-file torrent cid.
pub struct BtFileRef {
pub infohash20: [u8; 20],
pub file_index: usize,
}
/// Is this a per-file torrent cid (`0x1001` / `0x1002`)?
pub fn is_btih_file(mscid: &MsCid) -> bool {
matches!(mscid.codec(), BTIH_V1_FILE_CODEC | BTIH_V2_FILE_CODEC)
}
/// Decode a `btih-v1-file` cid into `(infohash, file_index)`; v2 is rejected
/// pending swarm support.
pub fn decode_btih_file(mscid: &MsCid) -> Result<BtFileRef> {
let infohash_len = match mscid.codec() {
BTIH_V1_FILE_CODEC => BT_V1_INFOHASH_LEN,
BTIH_V2_FILE_CODEC => {
let _ = BT_V2_INFOHASH_LEN; // documents the v2 digest shape
anyhow::bail!("btih-v2-file CIDs are not yet supported by the fetch tier");
}
other => anyhow::bail!("not a btih-file CID (codec 0x{other:x})"),
};
let digest = mscid.hash().digest();
anyhow::ensure!(
digest.len() > infohash_len,
"btih-file digest is {} bytes, need > {} (infohash + file-index varint)",
digest.len(),
infohash_len
);
let (ih, idx_bytes) = digest.split_at(infohash_len);
let infohash20 = <[u8; 20]>::try_from(ih)
.map_err(|_| anyhow::anyhow!("btih-file infohash slice is not 20 bytes"))?;
let (file_index, consumed) = read_uvarint(idx_bytes)?;
anyhow::ensure!(
consumed == idx_bytes.len(),
"btih-file digest has {} trailing byte(s) after the file-index varint",
idx_bytes.len() - consumed
);
let file_index = usize::try_from(file_index)
.map_err(|_| anyhow::anyhow!("btih-file index {file_index} overflows usize"))?;
Ok(BtFileRef { infohash20, file_index })
}
/// Minimal unsigned-LEB128 reader → `(value, bytes_consumed)`. Rejects truncated
/// and overflowing encodings.
pub fn read_uvarint(bytes: &[u8]) -> Result<(u64, usize)> {
let mut result: u64 = 0;
let mut shift: u32 = 0;
for (i, &b) in bytes.iter().enumerate() {
anyhow::ensure!(shift < 64, "varint overflows u64");
result |= u64::from(b & 0x7f) << shift;
if b & 0x80 == 0 {
return Ok((result, i + 1));
}
shift += 7;
}
anyhow::bail!("varint is truncated (no terminating byte)")
}
/// Is this cid a torrent of either shape?
pub fn is_torrent_cid(mscid: &MsCid) -> bool {
is_btih(mscid) || is_btih_file(mscid)
}
/// A torrent cid as `(infohash, file_index)`; a whole-torrent cid is file 0.
/// `None` for anything else.
pub fn torrent_ref_of_cid(cid: &str) -> Option<([u8; 20], usize)> {
let mscid = parse_record_cid(cid).ok()?;
if let Ok(r) = decode_btih_file(&mscid) {
return Some((r.infohash20, r.file_index));
}
infohash20_from_mscid(&mscid).ok().map(|ih| (ih, 0))
}
/// A torrent cid's v1 infohash (whole-torrent and per-file share it).
pub fn infohash_of_cid(cid: &str) -> Option<[u8; 20]> {
torrent_ref_of_cid(cid).map(|(ih, _)| ih)
}
/// Lowercase hex.
pub fn to_hex(bytes: &[u8]) -> String {
use std::fmt::Write;
let mut s = String::with_capacity(bytes.len() * 2);
for b in bytes {
let _ = write!(s, "{b:02x}");
}
s
}
/// Lowercase hex of a v1 infohash — the magnet-cache and seed-unit key.
pub fn infohash20_hex(infohash20: &[u8; 20]) -> String {
to_hex(infohash20)
}
/// Parse a 40-char hex infohash.
pub fn infohash20_from_hex(s: &str) -> Option<[u8; 20]> {
if s.len() != 40 {
return None;
}
let mut out = [0u8; 20];
for (i, pair) in s.as_bytes().chunks(2).enumerate() {
let hi = (pair[0] as char).to_digit(16)?;
let lo = (pair[1] as char).to_digit(16)?;
out[i] = ((hi << 4) | lo) as u8;
}
Some(out)
}
/// Best-effort MIME from a filename extension; `application/octet-stream` when
/// unknown so the browser sniffs.
pub fn content_type_from_name(name: &str) -> String {
let ext = name.rsplit('.').next().unwrap_or("").to_ascii_lowercase();
let ct = match ext.as_str() {
"mkv" => "video/x-matroska",
"mp4" | "m4v" => "video/mp4",
"webm" => "video/webm",
"avi" => "video/x-msvideo",
"mov" => "video/quicktime",
"ts" | "mts" | "m2ts" => "video/mp2t",
"mpg" | "mpeg" => "video/mpeg",
"wmv" => "video/x-ms-wmv",
"flv" => "video/x-flv",
"ogv" => "video/ogg",
"mp3" => "audio/mpeg",
"flac" => "audio/flac",
"aac" => "audio/aac",
"ogg" | "oga" => "audio/ogg",
"wav" => "audio/wav",
"pdf" => "application/pdf",
_ => "application/octet-stream",
};
ct.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn base32_lower(input: &[u8]) -> String {
const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
let mut out = String::new();
let mut buffer: u64 = 0;
let mut bits: u32 = 0;
for &b in input {
buffer = (buffer << 8) | b as u64;
bits += 8;
while bits >= 5 {
bits -= 5;
out.push(ALPHABET[((buffer >> bits) & 0x1F) as usize] as char);
}
}
if bits > 0 {
out.push(ALPHABET[((buffer << (5 - bits)) & 0x1F) as usize] as char);
}
out
}
fn make_cid(codec: u8, mh_code: u8, digest: &[u8]) -> MsCid {
let mut wire = vec![0x01u8, codec, mh_code, digest.len() as u8];
wire.extend_from_slice(digest);
MsCid::from_str(&format!("b{}", base32_lower(&wire))).expect("cid parse")
}
fn write_uvarint(mut v: u64, out: &mut Vec<u8>) {
loop {
let mut byte = (v & 0x7f) as u8;
v >>= 7;
if v != 0 {
byte |= 0x80;
}
out.push(byte);
if v == 0 {
break;
}
}
}
fn make_cid_varint(codec: u64, mh_code: u64, digest: &[u8]) -> MsCid {
let mut wire = vec![0x01u8];
write_uvarint(codec, &mut wire);
write_uvarint(mh_code, &mut wire);
write_uvarint(digest.len() as u64, &mut wire);
wire.extend_from_slice(digest);
MsCid::from_str(&format!("b{}", base32_lower(&wire))).expect("cid parse")
}
fn bt_file_digest(infohash: &[u8], index: u64) -> Vec<u8> {
let mut d = infohash.to_vec();
write_uvarint(index, &mut d);
d
}
fn v1_file_cid(infohash: &[u8; 20], index: u64) -> MsCid {
make_cid_varint(BTIH_V1_FILE_CODEC, BTIH_V1_FILE_CODEC, &bt_file_digest(infohash, index))
}
#[test]
fn btih_cid_is_recognised_and_decodes() {
let infohash = [7u8; 20];
let cid = make_cid(0x55, MH_SHA1 as u8, &infohash);
assert!(is_btih(&cid));
assert_eq!(infohash20_from_mscid(&cid).unwrap(), infohash);
}
#[test]
fn ipfs_sha256_cid_is_not_btih() {
let cid = make_cid(0x55, 0x12, &[0u8; 32]);
assert!(!is_btih(&cid));
assert!(infohash20_from_mscid(&cid).is_err());
}
#[test]
fn dagpb_cid_is_not_btih() {
assert!(!is_btih(&make_cid(0x70, 0x12, &[0u8; 32])));
}
#[test]
fn btih_v1_file_cid_is_recognised_and_decodes() {
let infohash = [7u8; 20];
let cid = v1_file_cid(&infohash, 4);
assert!(is_btih_file(&cid));
assert!(!is_btih(&cid));
let r = decode_btih_file(&cid).unwrap();
assert_eq!(r.infohash20, infohash);
assert_eq!(r.file_index, 4);
}
#[test]
fn btih_v1_file_index_zero_and_multibyte_roundtrip() {
assert_eq!(decode_btih_file(&v1_file_cid(&[0xab; 20], 0)).unwrap().file_index, 0);
assert_eq!(decode_btih_file(&v1_file_cid(&[0x11; 20], 200)).unwrap().file_index, 200);
}
#[test]
fn btih_v2_file_cid_recognised_but_rejected() {
let cid = make_cid_varint(BTIH_V2_FILE_CODEC, BTIH_V2_FILE_CODEC, &bt_file_digest(&[0x22; 32], 1));
assert!(is_btih_file(&cid));
assert!(decode_btih_file(&cid).is_err());
}
#[test]
fn whole_torrent_cid_is_index_zero_and_not_a_file_cid() {
let infohash = [9u8; 20];
let cid = make_cid(0x55, MH_SHA1 as u8, &infohash);
assert!(is_btih(&cid));
assert!(!is_btih_file(&cid));
assert!(decode_btih_file(&cid).is_err());
assert_eq!(torrent_ref_of_cid(&cid.to_string()), Some((infohash, 0)));
}
#[test]
fn decode_btih_file_rejects_missing_index_varint() {
let cid = make_cid_varint(BTIH_V1_FILE_CODEC, BTIH_V1_FILE_CODEC, &[3u8; 20]);
assert!(decode_btih_file(&cid).is_err());
}
#[test]
fn read_uvarint_rejects_truncated_and_reads_multibyte() {
assert!(read_uvarint(&[0x80]).is_err());
assert_eq!(read_uvarint(&[0xAC, 0x02]).unwrap(), (300, 2));
assert_eq!(read_uvarint(&[0x05, 0xFF]).unwrap(), (5, 1));
}
#[test]
fn torrent_ref_of_a_file_cid_keeps_its_index() {
let cid = v1_file_cid(&[5u8; 20], 3).to_string();
assert_eq!(torrent_ref_of_cid(&cid), Some(([5u8; 20], 3)));
assert_eq!(infohash_of_cid(&cid), Some([5u8; 20]));
assert_eq!(torrent_ref_of_cid("not a cid"), None);
}
#[test]
fn hex_round_trips() {
let ih = [0xabu8; 20];
assert_eq!(infohash20_from_hex(&infohash20_hex(&ih)), Some(ih));
assert_eq!(infohash20_from_hex("zz"), None);
}
#[test]
fn parse_rejects_token_form_and_cidv0() {
assert!(parse_record_cid("midhash256:bafy").is_err());
assert!(parse_record_cid("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG").is_err());
}
#[test]
fn content_type_guesses_common_media() {
assert_eq!(content_type_from_name("movie.mkv"), "video/x-matroska");
assert_eq!(content_type_from_name("clip.MP4"), "video/mp4");
assert_eq!(content_type_from_name("noext"), "application/octet-stream");
}
}
+336
View File
@@ -0,0 +1,336 @@
//! The hull side of the contract: an HTTP client for one transport plugin.
//!
//! [`RemoteTransport`] covers the common routes and gives typed helpers for a
//! plugin's extra ones (`get_json`, `post_json`, …). Byte responses come back as
//! an axum [`Response`] whose body streams straight from the plugin, so the hull
//! relays a multi-GB range without buffering it.
use std::time::Duration;
use axum::body::Body;
use axum::http::{HeaderMap, HeaderName, HeaderValue, Method, StatusCode};
use axum::response::{IntoResponse, Response};
use serde::de::DeserializeOwned;
use serde::Serialize;
use super::dto::{
Deleted, Health, Job, Jobs, Manifest, ReconcileReport, ReconcileRequest,
};
use super::error::ApiError;
use super::focus::FocusSnapshot;
/// Request headers never forwarded to a plugin (hop-by-hop, or describing the
/// client's connection to the hull rather than the hull's to the plugin).
const HOP_BY_HOP: &[&str] = &[
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"host",
"content-length",
];
/// Client for one plugin at `base` (e.g. `http://metashare-torrent:3000`).
#[derive(Clone, Debug)]
pub struct RemoteTransport {
id: String,
base: String,
http: reqwest::Client,
/// For control calls (jobs, focus, reconcile). `/raw` has no overall
/// timeout — a body can legitimately stream for the length of a film.
control_timeout: Duration,
}
impl RemoteTransport {
/// `id` names the tier in error messages (`torrent`, `nzb`, `ipfs`).
pub fn new(id: impl Into<String>, base: impl Into<String>, http: reqwest::Client) -> Self {
Self {
id: id.into(),
base: base.into().trim_end_matches('/').to_string(),
http,
control_timeout: Duration::from_secs(30),
}
}
pub fn with_control_timeout(mut self, t: Duration) -> Self {
self.control_timeout = t;
self
}
pub fn id(&self) -> &str {
&self.id
}
pub fn base(&self) -> &str {
&self.base
}
pub fn url(&self, path: &str) -> String {
format!("{}{}", self.base, path)
}
fn unreachable(&self, e: impl std::fmt::Display) -> ApiError {
ApiError::Upstream(format!("{} plugin unreachable: {e}", self.id))
}
// ---- common routes ---------------------------------------------------
pub async fn manifest(&self) -> Result<Manifest, ApiError> {
self.get_json("/manifest").await
}
pub async fn health(&self) -> Result<Health, ApiError> {
let r = self
.http
.get(self.url("/health"))
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
r.json::<Health>().await.map_err(|e| self.unreachable(e))
}
/// `GET /raw/:cid`, forwarding the client's request headers plus `extra`.
/// Returns the plugin's response as-is (any status), with its body streaming.
pub async fn raw(
&self,
cid: &str,
client_headers: &HeaderMap,
extra: &[(&str, String)],
) -> Result<Response, ApiError> {
let mut req = self.http.get(self.url(&format!("/raw/{cid}")));
req = req.headers(forwardable(client_headers));
for (k, v) in extra {
req = req.header(*k, v);
}
let resp = req.send().await.map_err(|e| self.unreachable(e))?;
Ok(relay(resp))
}
pub async fn jobs(&self) -> Result<Vec<Job>, ApiError> {
Ok(self.get_json::<Jobs>("/jobs").await?.jobs)
}
pub async fn delete(&self, cid: &str, keep: &[String]) -> Result<Deleted, ApiError> {
let r = self
.http
.delete(self.url(&format!("/jobs/{cid}")))
.query(&[("keep", keep.join(","))])
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
decode(r).await
}
pub async fn reconcile(&self, req: &ReconcileRequest) -> Result<ReconcileReport, ApiError> {
// A reconcile can delete many units; give it more room than a poll.
let r = self
.http
.post(self.url("/reconcile"))
.json(req)
.timeout(self.control_timeout * 10)
.send()
.await
.map_err(|e| self.unreachable(e))?;
decode(r).await
}
pub async fn push_focus(&self, snap: &FocusSnapshot) -> Result<(), ApiError> {
let r = self
.http
.put(self.url("/focus"))
.json(snap)
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
if r.status().is_success() {
Ok(())
} else {
Err(ApiError::Upstream(format!("{} plugin /focus: {}", self.id, r.status())))
}
}
/// `PUT /manifests/:cid`: hand a manifest-consuming plugin (contract 2) the
/// manifest a pointer resolved to, after it answered `428` +
/// [`super::dto::HDR_NEEDS_MANIFEST`].
pub async fn push_manifest(&self, cid: &str, manifest: bytes::Bytes) -> Result<(), ApiError> {
let r = self
.http
.put(self.url(&format!("/manifests/{cid}")))
.body(manifest)
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
if r.status().is_success() {
Ok(())
} else {
Err(error_from(r).await)
}
}
// ---- typed helpers for extra routes ------------------------------------
pub async fn get_json<T: DeserializeOwned>(&self, path: &str) -> Result<T, ApiError> {
let r = self
.http
.get(self.url(path))
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
decode(r).await
}
pub async fn post_json<B: Serialize + ?Sized, T: DeserializeOwned>(
&self,
path: &str,
body: &B,
) -> Result<T, ApiError> {
self.post_json_timeout(path, body, self.control_timeout).await
}
pub async fn post_json_timeout<B: Serialize + ?Sized, T: DeserializeOwned>(
&self,
path: &str,
body: &B,
timeout: Duration,
) -> Result<T, ApiError> {
let r = self
.http
.post(self.url(path))
.json(body)
.timeout(timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
decode(r).await
}
pub async fn put_json<B: Serialize + ?Sized>(&self, path: &str, body: &B) -> Result<(), ApiError> {
let r = self
.http
.put(self.url(path))
.json(body)
.timeout(self.control_timeout)
.send()
.await
.map_err(|e| self.unreachable(e))?;
if r.status().is_success() {
Ok(())
} else {
Err(error_from(r).await)
}
}
/// Forward an arbitrary request and relay the response verbatim — for the
/// public routes a plugin now answers (`/ipfs/:cid`, `/api/peers`, …).
pub async fn forward(
&self,
method: Method,
path_and_query: &str,
client_headers: &HeaderMap,
body: Option<bytes::Bytes>,
) -> Result<Response, ApiError> {
let mut req = self
.http
.request(method, self.url(path_and_query))
.headers(forwardable(client_headers));
if let Some(b) = body {
req = req.body(b);
}
let resp = req.send().await.map_err(|e| self.unreachable(e))?;
Ok(relay(resp))
}
}
/// Client request headers worth forwarding to a plugin.
fn forwardable(h: &HeaderMap) -> reqwest::header::HeaderMap {
let mut out = reqwest::header::HeaderMap::new();
for (k, v) in h {
if HOP_BY_HOP.contains(&k.as_str()) {
continue;
}
if let (Ok(k), Ok(v)) = (
reqwest::header::HeaderName::from_bytes(k.as_str().as_bytes()),
reqwest::header::HeaderValue::from_bytes(v.as_bytes()),
) {
out.append(k, v);
}
}
out
}
/// A plugin response as an axum response: same status, same headers (minus
/// hop-by-hop), body streamed.
pub fn relay(resp: reqwest::Response) -> Response {
let status = StatusCode::from_u16(resp.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
let mut headers = HeaderMap::new();
for (k, v) in resp.headers() {
let name = k.as_str();
if matches!(name, "connection" | "keep-alive" | "transfer-encoding" | "trailer" | "upgrade") {
continue;
}
if let (Ok(k), Ok(v)) = (
HeaderName::from_bytes(name.as_bytes()),
HeaderValue::from_bytes(v.as_bytes()),
) {
headers.append(k, v);
}
}
let body = Body::from_stream(resp.bytes_stream());
let mut out = (status, body).into_response();
*out.headers_mut() = headers;
out
}
async fn decode<T: DeserializeOwned>(r: reqwest::Response) -> Result<T, ApiError> {
if !r.status().is_success() {
return Err(error_from(r).await);
}
r.json::<T>()
.await
.map_err(|e| ApiError::Upstream(format!("plugin returned malformed JSON: {e}")))
}
/// Map a plugin error response back onto [`ApiError`], keeping its message so
/// a relayed error reads the same as the in-process one did.
pub async fn error_from(r: reqwest::Response) -> ApiError {
let status = r.status().as_u16();
let msg = r
.json::<serde_json::Value>()
.await
.ok()
.and_then(|v| v.get("error").and_then(|e| e.as_str()).map(str::to_string))
.unwrap_or_else(|| format!("plugin error {status}"));
match status {
400 => ApiError::BadRequest(msg),
404 => ApiError::NotFound,
504 => ApiError::GatewayTimeout(msg),
503 if msg == "swarm task is gone" => ApiError::SwarmGone,
_ => ApiError::Upstream(msg),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hop_by_hop_headers_are_not_forwarded() {
let mut h = HeaderMap::new();
h.insert("range", HeaderValue::from_static("bytes=0-1"));
h.insert("host", HeaderValue::from_static("x"));
h.insert("connection", HeaderValue::from_static("close"));
let f = forwardable(&h);
assert!(f.contains_key("range"));
assert!(!f.contains_key("host"));
assert!(!f.contains_key("connection"));
}
}
+347
View File
@@ -0,0 +1,347 @@
//! A transport plugin's own settings — the "config plane".
//!
//! Same model as a feeder's (`crate::config`): the plugin declares a
//! [`ConfigSchema`], keeps its values in `<state_dir>/config.json`, and serves a
//! schema-driven page that meta-share's dashboard embeds through its plugin
//! proxy (`/api/plugins/:name/ui/config`). Keys stay on the plugin; the hull
//! carries no protocol-specific settings.
//!
//! Routes (mounted by [`serve`](super::serve) when
//! [`TransportPlugin::config`](super::plugin::TransportPlugin::config) is
//! `Some`):
//!
//! - `GET /config` → the page ([`CONFIG_PAGE_HTML`]); every call it makes
//! is relative to its own path, so the proxy prefix is irrelevant.
//! - `GET /config/schema` → [`ConfigSchema`].
//! - `GET /config/values` → effective values, secrets redacted to `<key>_set`.
//! - `PUT /config/values` → merge (blank secret keeps the stored one), persist
//! atomically, then exit so the container restarts on the new values
//! (`{"saved":true,"reloading":true}`). Plugins read their config once at
//! boot — a restart is the one apply path that is right for every one of them.
//!
//! **Effective values** = `config.json` if it exists and parses as an object,
//! else the `seed` the plugin derived from its env (its pre-config-plane
//! settings source). A plugin deserializes [`ConfigPlane::effective`] into its
//! own typed struct with serde defaults, so a missing key is never fatal.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::{Html, IntoResponse, Response};
use axum::routing::get;
use axum::{Json, Router};
use serde_json::Value;
use tracing::{info, warn};
use crate::config::{merge, redact, ConfigSchema, CONFIG_PAGE_HTML};
/// Env var naming the plugin's own writable state dir.
pub const STATE_DIR_ENV: &str = "META_SHARE_PLUGIN_STATE_DIR";
/// Default state dir (the plugin app's own `AppData/<app>/state` mount).
pub const DEFAULT_STATE_DIR: &str = "/state";
/// File name of the stored values inside the state dir.
pub const CONFIG_FILE: &str = "config.json";
/// The plugin's state dir from [`STATE_DIR_ENV`], else [`DEFAULT_STATE_DIR`].
pub fn state_dir_from_env() -> PathBuf {
std::env::var(STATE_DIR_ENV)
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_STATE_DIR.to_string())
.into()
}
/// One plugin's schema + stored values.
pub struct ConfigPlane {
schema: ConfigSchema,
path: PathBuf,
seed: Value,
restart_on_save: bool,
}
impl ConfigPlane {
/// `seed` is what the plugin would run with if no `config.json` existed
/// (normally its env-derived settings, serialized).
pub fn new(schema: ConfigSchema, state_dir: &Path, seed: Value) -> Self {
Self {
schema,
path: state_dir.join(CONFIG_FILE),
seed,
restart_on_save: true,
}
}
/// Don't exit after a save (tests, or a plugin that hot-applies).
pub fn without_restart(mut self) -> Self {
self.restart_on_save = false;
self
}
pub fn schema(&self) -> &ConfigSchema {
&self.schema
}
pub fn path(&self) -> &Path {
&self.path
}
/// The stored values, if `config.json` exists and is a JSON object.
pub fn stored(&self) -> Option<Value> {
let bytes = std::fs::read(&self.path).ok()?;
match serde_json::from_slice::<Value>(&bytes) {
Ok(v @ Value::Object(_)) => Some(v),
Ok(_) | Err(_) => {
warn!(path = %self.path.display(), "plugin config is not a JSON object; using the seed");
None
}
}
}
/// Stored values, else the seed.
pub fn effective(&self) -> Value {
self.stored().unwrap_or_else(|| self.seed.clone())
}
/// Persist `values` atomically (tmp + rename), creating the state dir.
pub fn save(&self, values: &Value) -> std::io::Result<()> {
if let Some(dir) = self.path.parent() {
std::fs::create_dir_all(dir)?;
}
let tmp = self.path.with_extension("json.tmp");
let body = serde_json::to_vec_pretty(values)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
std::fs::write(&tmp, body)?;
std::fs::rename(&tmp, &self.path)
}
/// Write `values` as the initial config **only** if no `config.json` exists
/// yet — the one-shot migration hook (e.g. the nzb plugin importing the
/// NNTP block meta-share used to own). Returns whether it wrote.
pub fn seed_file_if_absent(&self, values: &Value) -> std::io::Result<bool> {
if self.path.exists() {
return Ok(false);
}
self.save(values)?;
Ok(true)
}
/// The `/config*` routes.
pub fn routes(self: Arc<Self>) -> Router {
Router::new()
.route("/config", get(page))
.route("/config/schema", get(schema))
.route("/config/values", get(values_get).put(values_put))
.with_state(self)
}
}
/// Lenient readers for a config object as the page writes it: an empty number
/// is `null`, an empty text is `""`, and a hand-edited file may hold a number as
/// a string. Each returns `None` for "not set" so the caller falls back to its
/// default.
pub mod read {
use serde_json::Value;
/// A non-blank, trimmed string.
pub fn text(v: &Value, key: &str) -> Option<String> {
match v.get(key)? {
Value::String(s) => Some(s.trim().to_string()).filter(|s| !s.is_empty()),
Value::Number(n) => Some(n.to_string()),
_ => None,
}
}
pub fn number(v: &Value, key: &str) -> Option<f64> {
match v.get(key)? {
Value::Number(n) => n.as_f64(),
Value::String(s) => s.trim().parse().ok(),
_ => None,
}
}
/// A non-negative integer (fractions truncate, negatives are "not set").
pub fn uint(v: &Value, key: &str) -> Option<u64> {
number(v, key).filter(|n| n.is_finite() && *n >= 0.0).map(|n| n as u64)
}
pub fn flag(v: &Value, key: &str) -> Option<bool> {
match v.get(key)? {
Value::Bool(b) => Some(*b),
Value::String(s) => match s.trim().to_ascii_lowercase().as_str() {
"1" | "true" | "yes" | "on" => Some(true),
"0" | "false" | "no" | "off" => Some(false),
_ => None,
},
_ => None,
}
}
/// A list of non-blank strings (a string is split on commas/newlines).
pub fn list(v: &Value, key: &str) -> Option<Vec<String>> {
let items: Vec<String> = match v.get(key)? {
Value::Array(a) => a.iter().filter_map(|x| x.as_str()).map(str::to_string).collect(),
Value::String(s) => s.split([',', '\n']).map(str::to_string).collect(),
_ => return None,
};
Some(items.into_iter().map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect())
}
}
async fn page() -> Html<&'static str> {
Html(CONFIG_PAGE_HTML)
}
async fn schema(State(p): State<Arc<ConfigPlane>>) -> Json<ConfigSchema> {
Json(p.schema.clone())
}
async fn values_get(State(p): State<Arc<ConfigPlane>>) -> Json<Value> {
Json(redact(&p.effective(), &p.schema))
}
async fn values_put(
State(p): State<Arc<ConfigPlane>>,
Json(incoming): Json<Value>,
) -> Result<Json<Value>, Response> {
if !incoming.is_object() {
return Err((StatusCode::BAD_REQUEST, "expected a JSON object").into_response());
}
let merged = merge(&p.effective(), &incoming, &p.schema);
p.save(&merged).map_err(|e| {
(StatusCode::INTERNAL_SERVER_ERROR, format!("write config: {e}")).into_response()
})?;
if !p.restart_on_save {
info!(path = %p.path.display(), "plugin config saved");
return Ok(Json(serde_json::json!({ "saved": true, "reloading": false })));
}
info!(path = %p.path.display(), "plugin config saved; restarting to apply");
// Let this response flush, then exit; `restart: unless-stopped` brings the
// plugin back and it reads config.json at boot.
tokio::spawn(async {
tokio::time::sleep(Duration::from_millis(800)).await;
std::process::exit(0);
});
Ok(Json(serde_json::json!({ "saved": true, "reloading": true })))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::ConfigField;
use axum::body::Body;
use axum::http::Request;
use tower::ServiceExt;
fn schema() -> ConfigSchema {
ConfigSchema {
fields: vec![
ConfigField::text("host", "Host"),
ConfigField::secret("pass", "Password"),
ConfigField::number("connections", "Connections"),
],
}
}
fn plane(dir: &Path) -> Arc<ConfigPlane> {
Arc::new(
ConfigPlane::new(schema(), dir, serde_json::json!({"host":"seed.example","connections":4}))
.without_restart(),
)
}
async fn call(app: Router, method: &str, uri: &str, body: Option<Value>) -> (StatusCode, Value) {
let req = Request::builder()
.method(method)
.uri(uri)
.header("content-type", "application/json")
.body(match body {
Some(b) => Body::from(serde_json::to_vec(&b).unwrap()),
None => Body::empty(),
})
.unwrap();
let r = app.oneshot(req).await.unwrap();
let status = r.status();
let bytes = axum::body::to_bytes(r.into_body(), 1 << 20).await.unwrap();
(status, serde_json::from_slice(&bytes).unwrap_or(Value::Null))
}
#[tokio::test]
async fn seed_until_saved_then_file_wins_and_secret_is_kept() {
let dir = tempfile::tempdir().unwrap();
let p = plane(dir.path());
assert_eq!(p.effective()["host"], "seed.example");
let (st, v) = call(Arc::clone(&p).routes(), "GET", "/config/values", None).await;
assert_eq!(st, StatusCode::OK);
assert_eq!(v["host"], "seed.example");
assert_eq!(v["pass_set"], false);
let (st, _) = call(
Arc::clone(&p).routes(),
"PUT",
"/config/values",
Some(serde_json::json!({"host":"news.example","pass":"s3cret"})),
)
.await;
assert_eq!(st, StatusCode::OK);
let stored = p.stored().unwrap();
assert_eq!(stored["host"], "news.example");
assert_eq!(stored["pass"], "s3cret");
assert_eq!(stored["connections"], 4, "absent field keeps the effective value");
// A blank secret keeps the stored one; the GET never leaks it.
call(Arc::clone(&p).routes(), "PUT", "/config/values", Some(serde_json::json!({"pass":""}))).await;
assert_eq!(p.stored().unwrap()["pass"], "s3cret");
let (_, v) = call(Arc::clone(&p).routes(), "GET", "/config/values", None).await;
assert!(v.get("pass").is_none());
assert_eq!(v["pass_set"], true);
}
#[tokio::test]
async fn schema_and_page_are_served() {
let dir = tempfile::tempdir().unwrap();
let p = plane(dir.path());
let (st, v) = call(Arc::clone(&p).routes(), "GET", "/config/schema", None).await;
assert_eq!(st, StatusCode::OK);
assert_eq!(v["fields"].as_array().unwrap().len(), 3);
let r = p
.routes()
.oneshot(Request::get("/config").body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(r.status(), StatusCode::OK);
}
#[test]
fn seed_file_if_absent_writes_once() {
let dir = tempfile::tempdir().unwrap();
let p = plane(dir.path());
assert!(p.seed_file_if_absent(&serde_json::json!({"host":"a"})).unwrap());
assert!(!p.seed_file_if_absent(&serde_json::json!({"host":"b"})).unwrap());
assert_eq!(p.effective()["host"], "a");
}
#[test]
fn readers_treat_blank_as_unset() {
let v = serde_json::json!({"a":"", "b":" x ", "n":null, "m":"7", "f":"off", "l":["a"," ",""], "s":"p, q"});
assert_eq!(read::text(&v, "a"), None);
assert_eq!(read::text(&v, "b").as_deref(), Some("x"));
assert_eq!(read::uint(&v, "n"), None);
assert_eq!(read::uint(&v, "m"), Some(7));
assert_eq!(read::flag(&v, "f"), Some(false));
assert_eq!(read::list(&v, "l"), Some(vec!["a".to_string()]));
assert_eq!(read::list(&v, "s"), Some(vec!["p".to_string(), "q".to_string()]));
assert_eq!(read::text(&v, "missing"), None);
}
#[test]
fn a_non_object_file_falls_back_to_the_seed() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(CONFIG_FILE), b"[1,2]").unwrap();
assert_eq!(plane(dir.path()).effective()["host"], "seed.example");
}
}
+308
View File
@@ -0,0 +1,308 @@
//! Wire types of the transport-plugin contract (hull ⇄ plugin, JSON over HTTP).
//!
//! Only what is common to every plugin lives here. A plugin's own extra routes
//! (ipfs `/share`, torrent `/commits`, …) carry their DTOs in the per-protocol
//! modules next to this one, so the hull and the plugin still compile against
//! one definition.
use serde::{Deserialize, Serialize};
use serde_json::Value;
/// Bumped on any breaking change to the common routes below. The hull refuses
/// a plugin whose `/manifest` reports a contract outside
/// [`MIN_CONTRACT`]`..=`[`CONTRACT_VERSION`].
///
/// - **2**: a plugin that consumes a manifest (nzb) no longer pulls it from the
/// hull. It answers `/raw` (and its verify route) with `428` +
/// [`HDR_NEEDS_MANIFEST`] until the hull has pushed one with
/// `PUT /manifests/:cid`.
pub const CONTRACT_VERSION: u32 = 2;
/// The oldest contract the hull still drives (contract-1 nzb plugins pull their
/// manifest from `/internal/nzb/manifest/:cid`, kept for one release).
pub const MIN_CONTRACT: u32 = 1;
/// `428` marker: this plugin needs the cid's manifest pushed before it can
/// answer. The hull resolves the pointer, `PUT`s `/manifests/:cid`, and retries.
pub const HDR_NEEDS_MANIFEST: &str = "x-metamesh-needs-manifest";
/// Why a pointer could not be resolved, on a non-2xx `/ipfs-tier/resolve`
/// answer: [`RESOLVE_NO_GATEWAY`], [`RESOLVE_UNCLAIMED`], [`RESOLVE_QUOTA`],
/// [`RESOLVE_NOT_FOUND`] or [`RESOLVE_UPSTREAM`].
pub const HDR_RESOLVE_ERROR: &str = "x-metamesh-resolve-error";
/// No gateway is known at all yet (discovery is eventual): retry.
pub const RESOLVE_NO_GATEWAY: &str = "no-gateway";
/// Gateways are known, but none claims this pointer's key.
pub const RESOLVE_UNCLAIMED: &str = "unclaimed";
/// The provider's quota is spent: retry after `Retry-After`.
pub const RESOLVE_QUOTA: &str = "quota";
/// Every claiming gateway answered not found (or nothing is stored and the
/// pointer cannot be redeemed).
pub const RESOLVE_NOT_FOUND: &str = "not-found";
/// A gateway failed, or answered bytes that did not verify.
pub const RESOLVE_UPSTREAM: &str = "upstream";
/// `X-MetaMesh-Lane`: the hull's lane decision for a `/raw` request.
pub const HDR_LANE: &str = "x-metamesh-lane";
/// `X-MetaMesh-Player`: forwarded verbatim from the client — marks a genuine
/// play (commits, provisional focus).
pub const HDR_PLAYER: &str = "x-metamesh-player";
/// `X-MetaMesh-Magnet`: the magnet the hull resolved for a torrent cid.
pub const HDR_MAGNET: &str = "x-metamesh-magnet";
/// Seed facts a plugin attaches to a successful `/raw` response, so the hull
/// records the seed row *before* it relays the body (the monolith wrote the
/// row inside the fetch, ahead of the first byte). Internal: the hull strips
/// every `x-metamesh-seed-*` header before relaying.
pub const HDR_SEED_NAME: &str = "x-metamesh-seed-name";
pub const HDR_SEED_SIZE: &str = "x-metamesh-seed-size";
/// Prefix of the seed headers the hull strips from a relayed response.
pub const INTERNAL_HEADER_PREFIX: &str = "x-metamesh-seed-";
/// `x-metamesh-meter: egress` on a `/raw` response: the hull floors this body
/// against the playback focus when it relays it. Set exactly where the
/// monolith metered (`focus::gated_stream`) — torrent ranges, Usenet file
/// reads, IPFS dag ranges — and *not* on single-block IPFS answers (posters),
/// which the monolith never throttled. Internal; stripped by the hull.
pub const HDR_METER: &str = "x-metamesh-meter";
pub const METER_EGRESS: &str = "egress";
/// Encode free text (a file name) for an internal header value: every byte
/// outside a conservative visible-ASCII set becomes `%XX`. Inverse:
/// [`header_text_decode`].
pub fn header_text_encode(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for b in s.bytes() {
if b.is_ascii_alphanumeric() || b" ._-()[]+,".contains(&b) {
out.push(b as char);
} else {
out.push_str(&format!("%{b:02X}"));
}
}
out
}
/// Inverse of [`header_text_encode`]. Invalid escapes pass through literally.
pub fn header_text_decode(s: &str) -> String {
let bytes = s.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' && i + 2 < bytes.len() {
let hi = (bytes[i + 1] as char).to_digit(16);
let lo = (bytes[i + 2] as char).to_digit(16);
if let (Some(hi), Some(lo)) = (hi, lo) {
out.push((hi * 16 + lo) as u8);
i += 3;
continue;
}
}
out.push(bytes[i]);
i += 1;
}
String::from_utf8_lossy(&out).into_owned()
}
/// `GET /manifest`.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct Manifest {
/// Protocol id: `torrent` | `nzb` | `ipfs` (a competing implementation of a
/// protocol reports the same id — that's what makes it a drop-in).
pub id: String,
/// Implementation name + version, for the dashboard.
pub implementation: String,
pub version: String,
pub contract: u32,
pub capabilities: Capabilities,
/// Serves a config plane (`/config`, `/config/schema`, `/config/values`).
/// Set by the harness from [`TransportPlugin::config`](super::plugin::TransportPlugin::config);
/// absent in older manifests.
#[serde(default)]
pub config: bool,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Capabilities {
/// Can fetch bytes for a cid (`/raw`).
pub fetch: bool,
/// Serves what it fetched to other peers (seeds).
pub share: bool,
}
/// `GET /health`.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Health {
pub ok: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub detail: Option<Value>,
}
impl Health {
pub fn ok() -> Self {
Self { ok: true, detail: None }
}
}
/// One unit of bytes a plugin holds — the eviction unit as the plugin knows it
/// (a torrent, an NZB job). `GET /jobs`.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct Job {
/// Stable unit key (a torrent's infohash hex, an nzb job's cid).
pub id: String,
/// The cids this unit answers for, as far as the plugin can name them.
#[serde(default)]
pub cids: Vec<String>,
/// Bytes on disk right now.
pub bytes_on_disk: u64,
/// Every selected byte is present.
pub complete: bool,
/// Downloading in the background on its own (exempt from eviction).
pub filling: bool,
/// Plugin-defined state label (`live`, `paused`, `streaming`, …).
#[serde(default)]
pub state: String,
/// Plugin-specific detail the hull renders opaquely (live transfer stats).
#[serde(default, skip_serializing_if = "Value::is_null")]
pub extra: Value,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct Jobs {
pub jobs: Vec<Job>,
}
/// `DELETE /jobs/:cid` query: the *other* cids of the same unit the hull still
/// wants (sibling files of one torrent). The plugin keeps those.
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct DeleteQuery {
/// Comma-separated cids.
#[serde(default)]
pub keep: String,
}
impl DeleteQuery {
pub fn keep_list(&self) -> Vec<String> {
self.keep
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect()
}
}
/// `DELETE /jobs/:cid` → the bytes actually unlinked, which is what the hull's
/// eviction budget credits.
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Deleted {
pub bytes_freed: u64,
}
/// `POST /reconcile`: delete every unit no wanted cid names, and any on-disk
/// leftover no unit owns older than `grace_secs`.
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct ReconcileRequest {
pub wanted: Vec<String>,
pub grace_secs: u64,
/// Only meaningful for plugins whose state lives in a lazily-started
/// session (torrent): bring it up even if nothing asked for bytes yet,
/// because persisted units are on disk and only the session can say whose.
#[serde(default)]
pub start_session: bool,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ReconcileReport {
/// Units deleted because nothing wanted them.
pub units_deleted: u64,
pub unit_bytes_freed: u64,
/// On-disk leftovers removed.
pub orphans_deleted: u64,
pub orphan_bytes_freed: u64,
/// The plugin couldn't tell what it owns (session down): nothing deleted.
#[serde(default)]
pub skipped: bool,
}
/// A file inside a promoted cache container.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct PromotedFile {
/// Path relative to the container directory.
pub rel: String,
/// Absolute path on the shared `/data` volume.
pub path: String,
pub size: u64,
}
/// Plugin → hull notifications, `POST {hull}/internal/events` (see
/// [`super::hull::HullClient`]). The hull acknowledges once it has acted, so a
/// plugin that must not proceed before the hull indexed something can wait.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Event {
/// The plugin now holds bytes under `cid`: the hull upserts a seed row.
Seed {
cid: String,
/// `torrent` | `ipfs` (the hull's `SeedKind`).
kind: String,
#[serde(default)]
name: String,
#[serde(default)]
size_bytes: u64,
/// The container the bytes live in, when there is one.
#[serde(default, skip_serializing_if = "Option::is_none")]
container: Option<String>,
},
/// A completed fetch was published as a cache container (`rename(2)` into
/// `cache/`). The hull indexes it in the material index.
Promoted {
container: String,
cids: Vec<String>,
files: Vec<PromotedFile>,
},
/// The content behind `cid` turned out to be an archive: the hull marks
/// the meta-core record `fileType=archive`.
Archive { cid: String },
/// Bytes behind these cids are gone: the hull drops their seed rows.
Vanished { cids: Vec<String> },
/// What the plugin knows about `cid` changed (a new availability verdict):
/// the hull drops its cached probe grade.
ProbeStale { cid: String },
/// A fetch finished and its file is complete at `path` (still under
/// `tmp/`). The hull indexes it for local-first reads and decides what to
/// do next (the Usenet → IPFS re-seed).
Ready {
cid: String,
path: String,
size: u64,
},
/// Any event type this hull doesn't know yet. Ignored.
#[serde(other)]
Unknown,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn header_text_round_trips_unicode_and_reserved() {
for s in ["Plain Name.mkv", "Été 100% \"quoted\"/x.mkv", "", "%", "a%2"] {
assert_eq!(header_text_decode(&header_text_encode(s)), s);
}
assert!(header_text_encode("é/").is_ascii());
}
#[test]
fn delete_query_splits_and_trims() {
let q = DeleteQuery { keep: " a, ,b,".into() };
assert_eq!(q.keep_list(), vec!["a".to_string(), "b".to_string()]);
assert!(DeleteQuery::default().keep_list().is_empty());
}
#[test]
fn events_are_tagged_and_unknown_tolerated() {
let e = Event::Archive { cid: "b".into() };
let j = serde_json::to_string(&e).unwrap();
assert_eq!(j, r#"{"type":"archive","cid":"b"}"#);
let u: Event = serde_json::from_str(r#"{"type":"from_the_future","x":1}"#).unwrap();
assert_eq!(u, Event::Unknown);
}
}
+60
View File
@@ -0,0 +1,60 @@
//! The one error type every transport HTTP surface answers with.
//!
//! Moved verbatim from meta-share `crates/meta-share/src/api/errors.rs`. It lives
//! here, not in the hull, because a plugin's error body is relayed to meta-share's
//! public API unchanged: the JSON shape `{ "error": "<message>" }` and the
//! status per variant are the contract every dashboard and peer depends on, and
//! the hull and its plugins must not be able to drift on it.
use axum::{http::StatusCode, response::IntoResponse, Json};
#[derive(Debug)]
pub enum ApiError {
BadRequest(String),
NotFound,
SwarmGone,
/// This peer is consumer-only (no `META_CORE_URL`) so it can't satisfy a
/// raw-bytes request directly. The cross-peer proxy turns this into a
/// clearer 502 with hint text; the direct route returns 503.
ConsumerOnly,
Upstream(String),
/// Gateway-side compute exceeded its deadline. Distinct from `Upstream` so
/// the dashboard can surface a clearer "the source fetcher is slow" message
/// instead of a generic 502.
GatewayTimeout(String),
}
impl IntoResponse for ApiError {
fn into_response(self) -> axum::response::Response {
let (status, msg) = match self {
ApiError::BadRequest(m) => (StatusCode::BAD_REQUEST, m),
ApiError::NotFound => (StatusCode::NOT_FOUND, "not found".into()),
ApiError::SwarmGone => (
StatusCode::SERVICE_UNAVAILABLE,
"swarm task is gone".into(),
),
ApiError::ConsumerOnly => (
StatusCode::SERVICE_UNAVAILABLE,
"this peer has no META_CORE_URL; raw bytes are only served by the source peer".into(),
),
ApiError::Upstream(m) => (StatusCode::BAD_GATEWAY, m),
ApiError::GatewayTimeout(m) => (StatusCode::GATEWAY_TIMEOUT, m),
};
(status, Json(serde_json::json!({ "error": msg }))).into_response()
}
}
impl std::fmt::Display for ApiError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ApiError::BadRequest(m) => write!(f, "bad request: {m}"),
ApiError::NotFound => write!(f, "not found"),
ApiError::SwarmGone => write!(f, "swarm task is gone"),
ApiError::ConsumerOnly => write!(f, "consumer-only peer"),
ApiError::Upstream(m) => write!(f, "upstream: {m}"),
ApiError::GatewayTimeout(m) => write!(f, "timeout: {m}"),
}
}
}
impl std::error::Error for ApiError {}
+114
View File
@@ -0,0 +1,114 @@
//! Serving a byte range straight off a file on the shared `/data` volume.
//!
//! The hull's local-first read and the usenet plugin's materialised/partial
//! file reads answer with the same headers, so they share this. Moved from
//! meta-share's `nzb::serve::serve_file_range`, minus the focus metering: the
//! caller meters the body (the hull wraps every response it serves, a plugin
//! doesn't meter its `/raw` at all — the hull does on relay).
use std::path::Path;
use axum::body::Body;
use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use tokio::io::{AsyncReadExt, AsyncSeekExt};
use tokio_util::io::ReaderStream;
use super::range::{parse_range_header, RangeOutcome};
/// Stream a complete on-disk file of `total` bytes, honouring a single
/// `Range:` (206) else 200; 416 for an unsatisfiable range.
///
/// `Err` carries the IO error that stopped the open/seek, so a caller can tell
/// "the file moved under us" apart from a served response.
pub async fn serve_file_range(
path: &Path,
total: u64,
range: Option<&HeaderValue>,
) -> std::io::Result<Response> {
let ctype = content_type(path);
match parse_range_header(range, total) {
RangeOutcome::Unsatisfiable => {
let mut headers = HeaderMap::new();
headers.insert(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes */{total}")).unwrap(),
);
Ok((StatusCode::RANGE_NOT_SATISFIABLE, headers).into_response())
}
RangeOutcome::Ok { start, end } => {
let len = end - start + 1;
let mut file = tokio::fs::File::open(path).await?;
file.seek(std::io::SeekFrom::Start(start)).await?;
let body = Body::from_stream(ReaderStream::new(file.take(len)));
let mut headers = HeaderMap::new();
headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ctype));
headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes"));
headers.insert(header::CONTENT_LENGTH, HeaderValue::from(len));
headers.insert(
header::CONTENT_RANGE,
HeaderValue::from_str(&format!("bytes {start}-{end}/{total}")).unwrap(),
);
Ok((StatusCode::PARTIAL_CONTENT, headers, body).into_response())
}
// Absent / Unsupported → full 200.
_ => {
let file = tokio::fs::File::open(path).await?;
let body = Body::from_stream(ReaderStream::new(file));
let mut headers = HeaderMap::new();
headers.insert(header::CONTENT_TYPE, HeaderValue::from_static(ctype));
headers.insert(header::ACCEPT_RANGES, HeaderValue::from_static("bytes"));
headers.insert(header::CONTENT_LENGTH, HeaderValue::from(total));
Ok((StatusCode::OK, headers, body).into_response())
}
}
}
/// Coarse content-type from extension — enough for the player to pick a demuxer.
pub fn content_type(path: &Path) -> &'static str {
match path
.extension()
.and_then(|e| e.to_str())
.map(|e| e.to_ascii_lowercase())
.as_deref()
{
Some("mkv") => "video/x-matroska",
Some("mp4") | Some("m4v") => "video/mp4",
Some("webm") => "video/webm",
Some("avi") => "video/x-msvideo",
Some("mov") => "video/quicktime",
Some("ts") | Some("m2ts") => "video/mp2t",
_ => "application/octet-stream",
}
}
#[cfg(test)]
mod tests {
use super::*;
async fn body(r: Response) -> Vec<u8> {
axum::body::to_bytes(r.into_body(), 1 << 20).await.unwrap().to_vec()
}
#[tokio::test]
async fn range_full_and_unsatisfiable() {
let p = std::env::temp_dir().join(format!("file-range-{}.mkv", std::process::id()));
std::fs::write(&p, b"0123456789").unwrap();
let r = serve_file_range(&p, 10, Some(&HeaderValue::from_static("bytes=2-4"))).await.unwrap();
assert_eq!(r.status(), StatusCode::PARTIAL_CONTENT);
assert_eq!(r.headers()[header::CONTENT_RANGE], "bytes 2-4/10");
assert_eq!(r.headers()[header::CONTENT_TYPE], "video/x-matroska");
assert_eq!(body(r).await, b"234");
let r = serve_file_range(&p, 10, None).await.unwrap();
assert_eq!(r.status(), StatusCode::OK);
assert_eq!(body(r).await, b"0123456789");
let r = serve_file_range(&p, 10, Some(&HeaderValue::from_static("bytes=20-"))).await.unwrap();
assert_eq!(r.status(), StatusCode::RANGE_NOT_SATISFIABLE);
assert_eq!(r.headers()[header::CONTENT_RANGE], "bytes */10");
let _ = std::fs::remove_file(&p);
assert_eq!(
serve_file_range(&p, 10, None).await.unwrap_err().kind(),
std::io::ErrorKind::NotFound
);
}
}
+464
View File
@@ -0,0 +1,464 @@
//! Playback focus, as a transport plugin sees it.
//!
//! meta-share's hull owns the focus *policy*: `POST/DELETE /api/playback/:cid`,
//! the leases, the TTL reaper, which sibling CIDs make up a title. A plugin only
//! needs to *obey* it: pause a torrent that isn't focused, park a background
//! fill, floor a non-focused ingress. So the hull pushes a [`FocusSnapshot`]
//! (`PUT /focus`) on every transition, plus periodically so a restarted plugin
//! recovers, and the plugin answers its gates from a local [`FocusView`].
//!
//! The read side mirrors meta-share's `focus::FocusGate` method for method, so
//! code moved out of meta-share keeps its call sites. Two things are
//! deliberately different from the in-process gate:
//!
//! - **No leases.** A view never expires an entry on its own; only the next
//! snapshot changes it. The hull's reaper is the single source of truth.
//! - **Per-process buckets.** `throttle_in`/`throttle_out` floor non-focused
//! traffic at `bg_rate_bytes` *per process*, where the monolith shared one
//! bucket across tiers. HTTP response bodies are still metered once, by the
//! hull, when it relays a plugin's `/raw` — plugins must not meter their own
//! `/raw` egress, or a peer pulling through the hull pays twice.
use std::collections::HashSet;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde::{Deserialize, Serialize};
use tokio::sync::watch;
/// How often a parked background task re-checks whether the focus cleared.
/// Same value as meta-share's `focus::PARK_POLL`.
pub const PARK_POLL: Duration = Duration::from_millis(750);
/// Burst allowance as a multiple of the per-second rate, floored at
/// [`MIN_BURST_BYTES`]. Same values as meta-share's `focus.rs`.
const BURST_SECS: u64 = 2;
const MIN_BURST_BYTES: u64 = 512 * 1024;
/// Which bandwidth lane a byte flow belongs to.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Lane {
/// Belongs to a title a viewer is playing right now — never gated.
Focused,
/// Everything else. Parked or throttled while any title is focused;
/// unrestricted otherwise.
Background,
}
impl Lane {
pub fn is_focused(self) -> bool {
matches!(self, Lane::Focused)
}
/// Wire value of the `X-MetaMesh-Lane` header.
pub fn as_header(self) -> &'static str {
match self {
Lane::Focused => "focused",
Lane::Background => "background",
}
}
/// Parse the `X-MetaMesh-Lane` header. An absent or unknown value is
/// `Focused`: a request the hull didn't classify must never be starved.
pub fn from_header(v: Option<&str>) -> Lane {
match v {
Some("background") => Lane::Background,
_ => Lane::Focused,
}
}
}
/// One focused title: its group key plus every identity it is reachable under.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct FocusTitle {
/// The record's canonical cid — the stable per-title key.
pub group: String,
/// Every sibling cid of the title.
#[serde(default)]
pub cids: Vec<String>,
/// The torrent infohashes among those siblings, lowercase hex, pre-decoded
/// by the hull so a torrent plugin never re-parses cids on a transition.
#[serde(default)]
pub infohashes: Vec<String>,
}
/// The whole focus state, pushed by the hull on `PUT /focus`.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FocusSnapshot {
/// Master switch (`META_SHARE_FOCUS_ENABLED`). Off → every gate is a no-op.
pub enabled: bool,
/// Bytes/s a non-focused flow is floored at. `0` → hard park.
pub bg_rate_bytes: u64,
#[serde(default)]
pub titles: Vec<FocusTitle>,
}
impl Default for FocusSnapshot {
fn default() -> Self {
Self {
enabled: true,
bg_rate_bytes: 10 * 1024,
titles: Vec::new(),
}
}
}
struct Inner {
titles: Vec<FocusTitle>,
cids: HashSet<String>,
infohashes: HashSet<[u8; 20]>,
}
/// A plugin's local replica of the hull's focus gate.
pub struct FocusView {
inner: Mutex<Inner>,
/// `titles.len()` as an atomic, so the resting state is one relaxed load.
n: AtomicUsize,
enabled: std::sync::atomic::AtomicBool,
rate: AtomicU64,
out_bucket: TokenBucket,
in_bucket: TokenBucket,
/// Bumped on every applied snapshot that changed something. A torrent
/// plugin watches it to re-run its pause/unpause pass.
changed: watch::Sender<u64>,
}
impl FocusView {
pub fn new() -> Arc<Self> {
let d = FocusSnapshot::default();
let (changed, _) = watch::channel(0);
let burst = (d.bg_rate_bytes * BURST_SECS).max(MIN_BURST_BYTES);
Arc::new(Self {
inner: Mutex::new(Inner {
titles: Vec::new(),
cids: HashSet::new(),
infohashes: HashSet::new(),
}),
n: AtomicUsize::new(0),
enabled: std::sync::atomic::AtomicBool::new(d.enabled),
rate: AtomicU64::new(d.bg_rate_bytes),
out_bucket: TokenBucket::new(d.bg_rate_bytes, burst),
in_bucket: TokenBucket::new(d.bg_rate_bytes, burst),
changed,
})
}
/// Replace the state with `snap`. Returns `true` when anything changed (and
/// notifies [`subscribe`](Self::subscribe)rs).
pub fn apply(&self, snap: FocusSnapshot) -> bool {
let mut cids = HashSet::new();
let mut infohashes = HashSet::new();
for t in &snap.titles {
cids.extend(t.cids.iter().cloned());
cids.insert(t.group.clone());
infohashes.extend(t.infohashes.iter().filter_map(|h| decode_hex20(h)));
}
let changed = {
let mut inner = self.inner.lock().unwrap();
let changed = inner.titles != snap.titles
|| self.enabled.load(Ordering::Relaxed) != snap.enabled
|| self.rate.load(Ordering::Relaxed) != snap.bg_rate_bytes;
inner.titles = snap.titles;
inner.cids = cids;
inner.infohashes = infohashes;
self.n.store(inner.titles.len(), Ordering::Relaxed);
changed
};
self.enabled.store(snap.enabled, Ordering::Relaxed);
if self.rate.swap(snap.bg_rate_bytes, Ordering::Relaxed) != snap.bg_rate_bytes {
let burst = (snap.bg_rate_bytes * BURST_SECS).max(MIN_BURST_BYTES);
self.out_bucket.reconfigure(snap.bg_rate_bytes, burst);
self.in_bucket.reconfigure(snap.bg_rate_bytes, burst);
}
if changed {
self.changed.send_modify(|g| *g += 1);
}
changed
}
/// The current state, as the hull last pushed it.
pub fn snapshot(&self) -> FocusSnapshot {
FocusSnapshot {
enabled: self.enabled.load(Ordering::Relaxed),
bg_rate_bytes: self.rate.load(Ordering::Relaxed),
titles: self.inner.lock().unwrap().titles.clone(),
}
}
/// Fires after every snapshot that changed something.
pub fn subscribe(&self) -> watch::Receiver<u64> {
self.changed.subscribe()
}
pub fn bg_rate_bytes(&self) -> u64 {
self.rate.load(Ordering::Relaxed)
}
/// Is *any* title focused?
pub fn any(&self) -> bool {
self.enabled.load(Ordering::Relaxed) && self.n.load(Ordering::Relaxed) > 0
}
pub fn count(&self) -> usize {
self.n.load(Ordering::Relaxed)
}
/// Which lane does a byte flow for `cid` belong to? Same rule as the hull:
/// nothing focused → everything is `Focused`.
pub fn lane(&self, cid: &str) -> Lane {
if !self.any() {
return Lane::Focused;
}
if self.inner.lock().unwrap().cids.contains(cid) {
Lane::Focused
} else {
Lane::Background
}
}
pub fn is_cid_focused(&self, cid: &str) -> bool {
self.inner.lock().unwrap().cids.contains(cid)
}
pub fn focused_infohashes(&self) -> HashSet<[u8; 20]> {
self.inner.lock().unwrap().infohashes.clone()
}
pub fn is_infohash_focused(&self, infohash: &[u8; 20]) -> bool {
self.inner.lock().unwrap().infohashes.contains(infohash)
}
/// Park a background task while any title is focused.
pub async fn wait_turn(&self, lane: Lane) {
if lane.is_focused() {
return;
}
while self.any() {
tokio::time::sleep(PARK_POLL).await;
}
}
/// [`wait_turn`](Self::wait_turn) with a deadline; `false` if the focus
/// outlasted `budget`.
pub async fn wait_turn_timeout(&self, lane: Lane, budget: Duration) -> bool {
if lane.is_focused() {
return true;
}
let deadline = Instant::now() + budget;
while self.any() {
if Instant::now() >= deadline {
return false;
}
tokio::time::sleep(PARK_POLL).await;
}
true
}
/// Floor a non-focused egress of `n` bytes (bitswap block service).
pub async fn throttle_out(&self, lane: Lane, n: u64) {
if lane.is_focused() || !self.any() {
return;
}
if self.bg_rate_bytes() == 0 {
self.wait_turn(lane).await;
return;
}
self.out_bucket.consume(n).await;
}
/// Floor a non-focused ingress of `n` bytes that a caller is waiting on.
pub async fn throttle_in(&self, lane: Lane, n: u64) {
if lane.is_focused() || !self.any() {
return;
}
if self.bg_rate_bytes() == 0 {
self.wait_turn(lane).await;
return;
}
self.in_bucket.consume(n).await;
}
}
/// Meter a byte stream against a view's egress floor. Same shape as
/// meta-share's `focus::gated_stream`.
pub fn gated_stream<S, B, E>(
stream: S,
focus: Arc<FocusView>,
lane: Lane,
) -> impl futures::Stream<Item = Result<B, E>>
where
S: futures::Stream<Item = Result<B, E>>,
B: AsRef<[u8]>,
{
use futures::StreamExt;
stream.then(move |item| {
let focus = Arc::clone(&focus);
async move {
if let Ok(bytes) = &item {
focus.throttle_out(lane, bytes.as_ref().len() as u64).await;
}
item
}
})
}
/// Lowercase hex of a 20-byte infohash — the wire form in [`FocusTitle`].
pub fn infohash_hex(ih: &[u8; 20]) -> String {
let mut s = String::with_capacity(40);
for b in ih {
s.push_str(&format!("{b:02x}"));
}
s
}
fn decode_hex20(s: &str) -> Option<[u8; 20]> {
if s.len() != 40 {
return None;
}
let mut out = [0u8; 20];
for (i, pair) in s.as_bytes().chunks(2).enumerate() {
let hi = (pair[0] as char).to_digit(16)?;
let lo = (pair[1] as char).to_digit(16)?;
out[i] = ((hi << 4) | lo) as u8;
}
Some(out)
}
/// Leaky bucket that always waits (never errors on `n > burst`). Copied from
/// meta-share's `focus.rs`, plus `reconfigure` for a rate pushed at runtime.
struct TokenBucket {
state: Mutex<BucketState>,
}
struct BucketState {
tokens: f64,
last: Instant,
rate: u64,
burst: u64,
}
impl TokenBucket {
fn new(rate: u64, burst: u64) -> Self {
Self {
state: Mutex::new(BucketState {
tokens: burst as f64,
last: Instant::now(),
rate,
burst,
}),
}
}
fn reconfigure(&self, rate: u64, burst: u64) {
let mut s = self.state.lock().unwrap();
s.rate = rate;
s.burst = burst;
s.tokens = s.tokens.min(burst as f64);
}
async fn consume(&self, n: u64) {
let wait = {
let mut s = self.state.lock().unwrap();
if s.rate == 0 || n == 0 {
return;
}
let now = Instant::now();
let elapsed = now.duration_since(s.last).as_secs_f64();
s.last = now;
s.tokens = (s.tokens + elapsed * s.rate as f64).min(s.burst as f64);
s.tokens -= n as f64;
if s.tokens >= 0.0 {
return;
}
Duration::from_secs_f64(-s.tokens / s.rate as f64)
};
tokio::time::sleep(wait).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
fn snap(titles: Vec<FocusTitle>) -> FocusSnapshot {
FocusSnapshot {
titles,
..Default::default()
}
}
#[test]
fn nothing_focused_means_every_flow_is_focused() {
let v = FocusView::new();
assert!(!v.any());
assert_eq!(v.lane("bafyanything"), Lane::Focused);
}
#[test]
fn siblings_and_group_are_focused_others_background() {
let v = FocusView::new();
v.apply(snap(vec![FocusTitle {
group: "g".into(),
cids: vec!["a".into(), "b".into()],
infohashes: vec![infohash_hex(&[7u8; 20])],
}]));
assert!(v.any());
assert_eq!(v.lane("a"), Lane::Focused);
assert_eq!(v.lane("g"), Lane::Focused);
assert_eq!(v.lane("z"), Lane::Background);
assert!(v.is_infohash_focused(&[7u8; 20]));
assert!(!v.is_infohash_focused(&[8u8; 20]));
}
#[test]
fn disabled_gate_focuses_nothing() {
let v = FocusView::new();
v.apply(FocusSnapshot {
enabled: false,
bg_rate_bytes: 1,
titles: vec![FocusTitle {
group: "g".into(),
..Default::default()
}],
});
assert!(!v.any());
assert_eq!(v.lane("z"), Lane::Focused);
}
#[test]
fn apply_reports_change_only_on_change() {
let v = FocusView::new();
let rx = v.subscribe();
let s = snap(vec![FocusTitle {
group: "g".into(),
..Default::default()
}]);
assert!(v.apply(s.clone()));
assert!(!v.apply(s));
assert_eq!(*rx.borrow(), 1);
assert!(v.apply(snap(vec![])));
assert_eq!(*rx.borrow(), 2);
}
#[test]
fn lane_header_round_trip_defaults_to_focused() {
assert_eq!(Lane::from_header(Some("background")), Lane::Background);
assert_eq!(Lane::from_header(Some("focused")), Lane::Focused);
assert_eq!(Lane::from_header(None), Lane::Focused);
assert_eq!(Lane::from_header(Some("junk")), Lane::Focused);
assert_eq!(Lane::from_header(Some(Lane::Background.as_header())), Lane::Background);
}
#[test]
fn snapshot_json_round_trips() {
let s = snap(vec![FocusTitle {
group: "g".into(),
cids: vec!["a".into()],
infohashes: vec![infohash_hex(&[1u8; 20])],
}]);
let back: FocusSnapshot = serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap();
assert_eq!(back, s);
}
}
+230
View File
@@ -0,0 +1,230 @@
//! Filesystem helpers shared by the hull and the plugins that own a corner of
//! the shared `/data` volume: free space, on-disk size, and the grace-period
//! removal both `reclaim` passes (cache containers in the hull, `bt-temp/` in
//! the torrent plugin) are built on. Moved verbatim from meta-share
//! `reclaim.rs` / `eviction.rs`.
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use tracing::warn;
/// Free space (bytes available to an unprivileged process) on the filesystem
/// backing `path`; `None` when the query fails or the platform isn't supported.
#[cfg(unix)]
pub fn available_bytes(path: &Path) -> Option<u64> {
use std::os::unix::ffi::OsStrExt;
let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?;
// SAFETY: `c_path` is a valid NUL-terminated path; `statvfs` only writes
// into the zeroed-out struct and returns 0 on success.
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
let rc = unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) };
if rc != 0 {
return None;
}
#[allow(clippy::unnecessary_cast)]
Some((stat.f_bavail as u64).saturating_mul(stat.f_frsize as u64))
}
#[cfg(not(unix))]
pub fn available_bytes(_path: &Path) -> Option<u64> {
None
}
/// Apparent bytes under `dir` (file lengths, subdirectories walked, symlinks
/// skipped). `0` when absent. Blocking.
pub fn dir_size_blocking(dir: &Path) -> u64 {
let mut total = 0u64;
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&d) else { continue };
for entry in rd.flatten() {
match entry.file_type() {
Ok(ft) if ft.is_dir() => stack.push(entry.path()),
Ok(ft) if ft.is_file() => {
if let Ok(meta) = entry.metadata() {
total = total.saturating_add(meta.len());
}
}
_ => {}
}
}
}
total
}
/// Bytes `path` actually occupies: allocated blocks, not apparent length (a
/// streaming-only torrent's file is sparse and its length is the whole film).
/// Blocking.
pub fn disk_usage_blocking(path: &Path) -> u64 {
let mut total = 0u64;
let mut stack = vec![path.to_path_buf()];
while let Some(p) = stack.pop() {
let Ok(meta) = std::fs::symlink_metadata(&p) else { continue };
if meta.is_dir() {
if let Ok(rd) = std::fs::read_dir(&p) {
stack.extend(rd.flatten().map(|e| e.path()));
}
} else if meta.is_file() {
total = total.saturating_add(allocated(&meta));
}
}
total
}
#[cfg(unix)]
fn allocated(meta: &std::fs::Metadata) -> u64 {
use std::os::unix::fs::MetadataExt;
meta.blocks().saturating_mul(512)
}
#[cfg(not(unix))]
fn allocated(meta: &std::fs::Metadata) -> u64 {
meta.len()
}
/// Top-level entries of `dir` that `is_owned` doesn't claim.
pub async fn unowned_entries(
dir: &Path,
is_owned: impl Fn(&std::ffi::OsStr) -> bool,
) -> Vec<(PathBuf, OsString)> {
let mut out = Vec::new();
let Ok(mut rd) = tokio::fs::read_dir(dir).await else {
return out;
};
while let Ok(Some(entry)) = rd.next_entry().await {
let name = entry.file_name();
if !is_owned(&name) {
out.push((entry.path(), name));
}
}
out
}
/// Outcome of [`remove_if_stale`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Removal {
Removed(u64),
TooYoung,
Failed,
}
/// Has nothing under `path` changed for `grace`? Unreadable → `false` (leave it).
pub async fn is_stale(path: &Path, grace: Duration, now: SystemTime) -> bool {
let p = path.to_path_buf();
let newest = tokio::task::spawn_blocking(move || newest_change_blocking(&p))
.await
.ok()
.flatten();
match newest {
Some(t) => now.duration_since(t).map(|age| age >= grace).unwrap_or(false),
None => false,
}
}
/// Remove a file or directory tree once nothing under it changed for `grace`.
/// Returns the bytes it occupied on disk.
pub async fn remove_if_stale(path: &Path, grace: Duration, now: SystemTime) -> Removal {
if !grace.is_zero() && !is_stale(path, grace, now).await {
return Removal::TooYoung;
}
let p = path.to_path_buf();
let res = tokio::task::spawn_blocking(move || {
let bytes = disk_usage_blocking(&p);
let r = match std::fs::symlink_metadata(&p) {
Ok(m) if m.is_dir() => std::fs::remove_dir_all(&p),
Ok(_) => std::fs::remove_file(&p),
Err(e) => Err(e),
};
r.map(|()| bytes)
})
.await;
match res {
Ok(Ok(b)) => Removal::Removed(b),
Ok(Err(e)) if e.kind() == std::io::ErrorKind::NotFound => Removal::Removed(0),
Ok(Err(e)) => {
warn!(path = %path.display(), error = %e, "reclaim: removal failed");
Removal::Failed
}
Err(_) => Removal::Failed,
}
}
/// The most recent modification or status change of `path` or anything under
/// it. ctime matters: a container renamed from `tmp/` into `cache/` keeps its
/// old mtime, and only ctime records the move.
pub fn newest_change_blocking(path: &Path) -> Option<SystemTime> {
let mut newest: Option<SystemTime> = None;
let mut stack = vec![path.to_path_buf()];
while let Some(p) = stack.pop() {
let Ok(meta) = std::fs::symlink_metadata(&p) else {
if p == path {
return None;
}
continue;
};
for t in [meta.modified().ok(), change_time(&meta)].into_iter().flatten() {
newest = Some(newest.map_or(t, |n| n.max(t)));
}
if meta.is_dir() {
if let Ok(rd) = std::fs::read_dir(&p) {
stack.extend(rd.flatten().map(|e| e.path()));
}
}
}
newest
}
#[cfg(unix)]
fn change_time(meta: &std::fs::Metadata) -> Option<SystemTime> {
use std::os::unix::fs::MetadataExt;
let secs = u64::try_from(meta.ctime()).ok()?;
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(secs))
}
#[cfg(not(unix))]
fn change_time(_meta: &std::fs::Metadata) -> Option<SystemTime> {
None
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn young_entries_survive_and_zero_grace_removes() {
let d = std::env::temp_dir().join(format!("fsutil-{}", std::process::id()));
std::fs::create_dir_all(d.join("a")).unwrap();
std::fs::write(d.join("a/f"), vec![1u8; 4096]).unwrap();
let now = SystemTime::now();
assert_eq!(
remove_if_stale(&d.join("a"), Duration::from_secs(3600), now).await,
Removal::TooYoung
);
assert!(matches!(
remove_if_stale(&d.join("a"), Duration::ZERO, now).await,
Removal::Removed(_)
));
assert!(!d.join("a").exists());
assert_eq!(remove_if_stale(&d.join("gone"), Duration::ZERO, now).await, Removal::Removed(0));
let _ = std::fs::remove_dir_all(&d);
}
#[tokio::test]
async fn unowned_lists_only_unclaimed_names() {
let d = std::env::temp_dir().join(format!("fsutil-own-{}", std::process::id()));
std::fs::create_dir_all(d.join("keep")).unwrap();
std::fs::create_dir_all(d.join("drop")).unwrap();
let got = unowned_entries(&d, |n| n == "keep").await;
assert_eq!(got.len(), 1);
assert_eq!(got[0].1, "drop");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn sizes_of_a_missing_dir_are_zero() {
assert_eq!(dir_size_blocking(Path::new("/definitely/not/here")), 0);
assert_eq!(disk_usage_blocking(Path::new("/definitely/not/here")), 0);
}
}
+276
View File
@@ -0,0 +1,276 @@
//! The plugin → hull direction: callbacks a plugin makes into meta-share.
//!
//! The hull serves these on a **separate internal listener**
//! (`META_SHARE_INTERNAL_LISTEN`, default `0.0.0.0:3001`) that no deployment
//! publishes: they reach meta-core's records and write to them, so they must
//! never sit behind the public perimeter. A plugin finds it at
//! `META_SHARE_HULL_URL` (default `http://metashare-app:3001`).
//!
//! Routes:
//!
//! - `POST /internal/events` ← [`Event`] → `204` once the hull acted.
//! - `GET /internal/records/:cid[?cached=1]` → [`RecordInfo`] (`404` = no
//! record). `cached=1` answers from the hull's record cache only (no
//! meta-core round trip) — for display names on a hot path.
//! - `GET /internal/nzb/manifest/:cid` → the `.nzb` bytes the cid resolves to,
//! with [`HDR_MANIFEST_SOURCE`] (`record` | `redeem`) and, for `record`,
//! [`HDR_MANIFEST_CID`]. `404` = not a Usenet cid; any other error carries
//! `{"error"}` — the manifest could not be redeemed or fetched. Parsing and
//! the `nzb-posting` digest check stay in the plugin.
//! - `GET /internal/network` → [`NetworkInfo`]: the hull's own endpoints a
//! plugin needs to know (the ipfs plugin announces `peer_url` in identify).
//! - `GET /internal/legacy/usenet` → the NNTP block meta-share's `settings.json`
//! held before the nzb plugin owned its settings (`404` = none). One-shot
//! migration source for the nzb plugin's config plane.
//! - `GET /health` → `200` once the hull has finished its boot steps (the
//! ipfs plugin waits on it before opening `ipfs/blocks.redb`).
//!
//! Why callbacks and not plugin-side logic: these need what only the hull has —
//! the record cache and meta-core, the gateway redeem, the IPFS tier. Plugins
//! never call each other; the hull routes every such edge.
use std::time::Duration;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use super::dto::Event;
/// Default hull callback base, matching the dev and store compose service name.
pub const DEFAULT_HULL_URL: &str = "http://metashare-app:3001";
/// Where a pointer's bytes came from: `store` (already held here, named by the
/// record's pointer field), `redeem` (a gateway), or `record` (contract-1
/// callback naming for `store`).
pub const HDR_MANIFEST_SOURCE: &str = "x-metamesh-manifest-source";
/// The content cid the pointer resolved to.
pub const HDR_MANIFEST_CID: &str = "x-metamesh-manifest-cid";
/// `.nzb` bytes as the hull resolved them.
#[derive(Clone, Debug)]
pub struct ManifestBytes {
pub bytes: bytes::Bytes,
/// `true` when redeemed through a gateway, `false` for the record pointer.
pub redeemed: bool,
pub manifest_cid: Option<String>,
}
/// What a plugin may need to know about a record.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct RecordInfo {
/// meta-core's hash id for the record (write-back key).
#[serde(default)]
pub hash_id: Option<String>,
/// The record's `fileName`.
#[serde(default)]
pub file_name: Option<String>,
/// The record's display title.
#[serde(default)]
pub title: Option<String>,
}
/// The hull's endpoints (`GET /internal/network`).
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct NetworkInfo {
/// meta-core base URL the hull talks to.
#[serde(default)]
pub meta_core_url: Option<String>,
/// The URL other peers reach this meta-share at (announced in identify).
#[serde(default)]
pub peer_url: Option<String>,
}
/// Plugin-side client for the hull's internal API. A client with no base URL
/// (`META_SHARE_HULL_URL=` empty, or [`HullClient::disabled`]) answers every call
/// with "not available", which is what unit tests and a standalone plugin get.
#[derive(Clone, Debug)]
pub struct HullClient {
base: Option<String>,
http: reqwest::Client,
}
impl HullClient {
pub fn new(base: Option<String>, http: reqwest::Client) -> Self {
let base = base
.map(|b| b.trim().trim_end_matches('/').to_string())
.filter(|b| !b.is_empty());
Self { base, http }
}
/// From `META_SHARE_HULL_URL`, defaulting to [`DEFAULT_HULL_URL`]; an
/// explicitly empty value disables callbacks.
pub fn from_env(http: reqwest::Client) -> Self {
let base = match std::env::var("META_SHARE_HULL_URL") {
Ok(v) => Some(v),
Err(_) => Some(DEFAULT_HULL_URL.to_string()),
};
Self::new(base, http)
}
pub fn disabled() -> Self {
Self::new(None, reqwest::Client::new())
}
pub fn is_enabled(&self) -> bool {
self.base.is_some()
}
fn url(&self, path: &str) -> Result<String, String> {
self.base
.as_ref()
.map(|b| format!("{b}{path}"))
.ok_or_else(|| "hull callbacks disabled (META_SHARE_HULL_URL unset)".to_string())
}
/// Notify the hull; returns once it has acted.
pub async fn emit(&self, ev: &Event) -> Result<(), String> {
let r = self
.http
.post(self.url("/internal/events")?)
.json(ev)
.timeout(Duration::from_secs(30))
.send()
.await
.map_err(|e| format!("hull unreachable: {e}"))?;
if r.status().is_success() {
Ok(())
} else {
Err(format!("hull refused event: {}", r.status()))
}
}
/// `GET` a JSON resource; `404` → `Ok(None)`.
pub async fn get_json<T: DeserializeOwned>(&self, path: &str, timeout: Duration) -> Result<Option<T>, String> {
let r = self
.http
.get(self.url(path)?)
.timeout(timeout)
.send()
.await
.map_err(|e| format!("hull unreachable: {e}"))?;
if r.status().as_u16() == 404 {
return Ok(None);
}
if !r.status().is_success() {
return Err(error_message(r).await);
}
r.json::<T>().await.map(Some).map_err(|e| format!("hull sent malformed JSON: {e}"))
}
/// `GET` raw bytes; `404` → `Ok(None)`, any other failure → the hull's
/// `{"error"}` message.
pub async fn get_bytes(&self, path: &str, timeout: Duration) -> Result<Option<bytes::Bytes>, String> {
let r = self
.http
.get(self.url(path)?)
.timeout(timeout)
.send()
.await
.map_err(|e| format!("hull unreachable: {e}"))?;
if r.status().as_u16() == 404 {
return Ok(None);
}
if !r.status().is_success() {
return Err(error_message(r).await);
}
r.bytes().await.map(Some).map_err(|e| format!("hull body: {e}"))
}
/// The `.nzb` behind a Usenet cid. `Ok(None)` = not a Usenet cid.
pub async fn nzb_manifest(&self, cid: &str, timeout: Duration) -> Result<Option<ManifestBytes>, String> {
let r = self
.http
.get(self.url(&format!("/internal/nzb/manifest/{cid}"))?)
.timeout(timeout)
.send()
.await
.map_err(|e| format!("hull unreachable: {e}"))?;
if r.status().as_u16() == 404 {
return Ok(None);
}
if !r.status().is_success() {
return Err(error_message(r).await);
}
let h = r.headers();
let redeemed = h
.get(HDR_MANIFEST_SOURCE)
.and_then(|v| v.to_str().ok())
.is_some_and(|v| v == "redeem");
let manifest_cid = h
.get(HDR_MANIFEST_CID)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let bytes = r.bytes().await.map_err(|e| format!("hull body: {e}"))?;
Ok(Some(ManifestBytes { bytes, redeemed, manifest_cid }))
}
/// [`RecordInfo`] for `cid` (record cache, else meta-core), or `None` when
/// the hull has no record.
pub async fn record(&self, cid: &str) -> Result<Option<RecordInfo>, String> {
self.get_json(&format!("/internal/records/{cid}"), Duration::from_secs(15)).await
}
/// The hull's endpoints.
pub async fn network(&self) -> Result<Option<NetworkInfo>, String> {
self.get_json("/internal/network", Duration::from_secs(5)).await
}
/// The NNTP block meta-share's own settings held before the nzb plugin had
/// a config plane; `Ok(None)` = there is none.
pub async fn legacy_usenet(&self) -> Result<Option<super::nzb::UsenetSettings>, String> {
self.get_json("/internal/legacy/usenet", Duration::from_secs(5)).await
}
/// Wait until the hull's internal listener answers `GET /health` with 2xx,
/// or `max` elapses. Returns whether it came up.
pub async fn wait_ready(&self, max: Duration) -> bool {
let Ok(url) = self.url("/health") else {
return false;
};
let deadline = tokio::time::Instant::now() + max;
loop {
if let Ok(r) = self.http.get(&url).timeout(Duration::from_secs(3)).send().await {
if r.status().is_success() {
return true;
}
}
if tokio::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(Duration::from_secs(1)).await;
}
}
/// [`RecordInfo`] from the hull's record cache only.
pub async fn cached_record(&self, cid: &str) -> Result<Option<RecordInfo>, String> {
self.get_json(&format!("/internal/records/{cid}?cached=1"), Duration::from_secs(5)).await
}
}
async fn error_message(r: reqwest::Response) -> String {
let status = r.status();
r.json::<serde_json::Value>()
.await
.ok()
.and_then(|v| v.get("error").and_then(|e| e.as_str()).map(str::to_string))
.unwrap_or_else(|| format!("hull error {status}"))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn a_disabled_client_refuses_without_network() {
let h = HullClient::disabled();
assert!(!h.is_enabled());
assert!(h.emit(&Event::Archive { cid: "x".into() }).await.is_err());
assert!(h.record("x").await.is_err());
}
#[test]
fn blank_base_disables() {
assert!(!HullClient::new(Some(" ".into()), reqwest::Client::new()).is_enabled());
assert!(HullClient::new(Some("http://h:3001/".into()), reqwest::Client::new()).is_enabled());
}
}
+167
View File
@@ -0,0 +1,167 @@
//! IPFS-specific half of the contract: the ipfs plugin's extra routes.
//!
//! The ipfs plugin owns the libp2p swarm (identify / kad / mdns / bitswap), the
//! blockstore (`ipfs/blocks.redb`: material blocks + leaf refs into files on the
//! shared volume), bitswap ingress and the peer directory. These routes are the
//! hull's facade onto that — each mirrors one blockstore / swarm call the hull
//! made in-process, so the hull's logic (seed rows, eviction, meta-core links,
//! probe grading) did not have to change shape.
//!
//! Public passthrough (the hull relays meta-share's own route verbatim):
//!
//! - `GET /ipfs/:cid` — the read-only IPFS gateway.
//!
//! Internal (all under `/ipfs-tier/`):
//!
//! | route | in-process equivalent |
//! |---|---|
//! | `GET peers` → [`PeersInfo`] | `swarm::Command::Peers` |
//! | `GET directory` → [`Directory`] | `swarm::PeerDirectory` (gateway discovery) |
//! | `GET resolve/:cid?hint=` → bytes | a [pointer](super::pointer)'s bytes: the stored copy `hint` names, else a gateway redeem (see below) |
//! | `GET local/:cid` (Range) → bytes / `404` | `files::bitswap::try_local_block` |
//! | `GET has/:cid` → [`Present`] | `block_store.get(cid).is_some()` |
//! | `GET complete/:cid` → [`Complete`] | `local_dag_complete` |
//! | `GET cat/:cid?max=` → bytes | `ipfs_walker::get_block` (+ dag-pb assemble) |
//! | `GET fetch/:cid?timeout_ms=` → block bytes | `block_store.get` → `bitswap_get_block` |
//! | `PUT block/:cid` ← bytes | `blockstore::put_ipfs_block` |
//! | `POST import` ← file bytes → [`Imported`] | `put_ipfs_blocks(compute_ipfs_blocks(..))` |
//! | `POST forget/:cid` | `api::seeds::remove_ipfs_blocks` |
//! | `POST forget-library` ← [`ForgetLibrary`] | `block_store.remove_library_dag` |
//! | `POST provide/:cid`, `POST unprovide/:cid` | `Command::Provide` / `StopProviding` |
//! | `POST drop-refs` ← [`DropRefs`] | `block_store.drop_refs_for_materials` |
//! | `GET first-leaf/:cid` → [`FirstLeaf`] | `block_store.first_leaf_backing` |
//! | `GET rel-path/:midhash` → [`RelPath`] | `resolver().rel_path_for` |
//! | `POST share/library` ← [`ShareLibrary`] → [`Shared`] | `ipfs_seed::seed_file_as_refs` |
//! | `POST share/container` ← [`ShareContainer`] → [`Shared`] | the seal / rebuild re-chunk |
//! | `GET stats/blockstore` | `/api/stats/blockstore` body |
//! | `GET debug/:cid` | `/api/debug/blockstore/:cid` body |
//!
//! Plugin → hull, over [`super::hull`]: [`Event::Seed`](super::Event::Seed)
//! (a bitswap fetch now seeds), [`Event::Promoted`](super::Event::Promoted) (an
//! ingress job was renamed into `cache/`), and `GET /internal/records/:cid` for
//! a display name.
//! **Resolving a pointer** (`GET resolve/:cid`) is the one route that reaches
//! outside the swarm: a pointer is answered only by a gateway holding the key,
//! over plain HTTP at the gateway's base URL. Gateways come from config (the
//! local one, pinned) and from the swarm directory; the fastest one whose redeem
//! claim covers the pointer is asked first. `2xx` carries the bytes plus
//! [`HDR_MANIFEST_SOURCE`](super::hull::HDR_MANIFEST_SOURCE) (`store` | `redeem`)
//! and [`HDR_MANIFEST_CID`](super::hull::HDR_MANIFEST_CID); a failure carries
//! [`HDR_RESOLVE_ERROR`](super::dto::HDR_RESOLVE_ERROR).
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct PeersInfo {
pub local_peer_id: String,
pub connected: Vec<String>,
}
/// One peer of the directory, with its freshness as *ages* (the hull rebuilds
/// its own `Instant`s from them).
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct DirectoryPeer {
pub peer_id: String,
#[serde(default)]
pub base_url: Option<String>,
/// Milliseconds since the gateway last advertised `nzbFetch`; `None` = not.
#[serde(default)]
pub nzb_fetch_age_ms: Option<u64>,
/// Milliseconds since the caps were fetched; `None` = never.
#[serde(default)]
pub caps_age_ms: Option<u64>,
/// The gateway's redeem claims, as the gateway serves them
/// (`{codec, field, hosts, sources}`).
#[serde(default)]
pub redeems: Vec<serde_json::Value>,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct Directory {
pub peers: Vec<DirectoryPeer>,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Present {
pub present: bool,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Complete {
pub complete: bool,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct Imported {
pub root: String,
pub size: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct ForgetLibrary {
pub root: String,
pub midhash: String,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct DropRefs {
/// `(container, rel)` pairs.
pub materials: Vec<(String, String)>,
}
/// Where a dag's first leaf points (the filestore's `LeafBacking`).
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum LeafBacking {
/// A ref into a file meta-core owns, by midhash256.
MetaCore(String),
/// A ref into a file in a cache container.
Cache,
/// The leaf's bytes are stored material (swarm-fetched, no file behind it).
Material,
/// The root or a node on the way down is absent.
Unknown,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct FirstLeaf {
pub backing: LeafBacking,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct RelPath {
pub rel: Option<String>,
}
/// Seed a meta-core library file as refs `(midhash, offset, len)`.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct ShareLibrary {
/// Path relative to the files volume (what meta-core's WebDAV serves).
pub rel: String,
pub midhash: String,
}
/// Seed a file in a cache container as refs `(container, rel, offset, len)`.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct ShareContainer {
pub container: String,
pub rel: String,
/// Absolute path on the shared volume.
pub path: String,
/// Verify the resulting dag is fully held before answering (the seal does;
/// the rebuild doesn't).
#[serde(default)]
pub verify_complete: bool,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct Shared {
pub root: String,
pub size: u64,
/// Leaf refs written.
pub refs: u64,
/// `verify_complete` was asked and the dag is fully held locally.
#[serde(default)]
pub complete: bool,
}
+230
View File
@@ -0,0 +1,230 @@
//! Content sniffing — what the bytes *are*, independent of what they are called.
//!
//! Every classifier in the usenet plugin's `archive` keys off the filename quoted in an NZB
//! subject. A fully obfuscated posting has no filename to key off, so all of them
//! fail **open**: `subject_is_par2` misses the parity volumes, `subject_is_rar`
//! misses the archive, and `pick_media`'s `best_video.or(best_any)` then elects
//! the largest arbitrary blob as "the movie". That blob is what gets served, and
//! — far worse — what the Usenet → IPFS re-seed hashes and publishes as
//! the record's content identity.
//!
//! Names are advisory. Bytes are not. This module is the byte half.
//!
//! The resolution order mirrors `filename-tool`'s `FileTypeConfigurable`
//! (the source of truth named by `METADATA_KEYS.md` §`fileType`): sniff the
//! content first, compare with the extension when there is one, and on
//! **disagreement return `undefined` rather than guessing** — a wrong `fileType`
//! routes a file to a handler that cannot open it.
/// A container format recognised from its leading bytes.
///
/// Deliberately *not* exhaustive over "every format that exists" — it only needs
/// to separate payload from wrapper from parity well enough to decide whether a
/// posting has resolved. [`Magic::Unknown`] is a first-class answer, and callers
/// must treat it as "cannot prove anything", never as "bad".
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Magic {
/// RAR 1.5–4.x — signature `52 61 72 21 1A 07 00`.
Rar4,
/// RAR 5.0+ — signature `52 61 72 21 1A 07 01 00`.
Rar5,
/// par2 recovery data. **Never payload**, whatever its size.
Par2,
SevenZip,
/// A zip that is not an EPUB (includes `.cbz`, which `METADATA_KEYS.md`
/// §`fileType` classifies as `archive`).
Zip,
/// A zip whose first entry is the stored `mimetype` = `application/epub+zip`.
/// Split from [`Magic::Zip`] because an EPUB **is** the document, not a
/// wrapper around one — three correct ebook postings on watch.nsl.sh are
/// this shape and a naive "zip ⇒ archive" rule condemns all of them.
Epub,
/// Palm database holding an ebook (`BOOK`/`TEXt` type at offset 60) — the
/// `CR!…`-named MobiPocket/eReader files an indexer hands back for a book.
PalmDoc,
Matroska,
Mp4,
/// RIFF container (AVI, and also WAV — the extension disambiguates).
Riff,
Pdf,
/// Nothing recognised. Not an error: see the type-level note above.
Unknown,
}
impl Magic {
/// The `METADATA_KEYS.md` §`fileType` value these bytes imply, or `None`
/// when the content says nothing.
///
/// [`Magic::Riff`] is deliberately `None`: RIFF is AVI *and* WAV, i.e. a
/// video and an audio type, and guessing between them is exactly the
/// "disagreement returns undefined" case the registry legislates.
pub fn file_type(self) -> Option<&'static str> {
match self {
Magic::Matroska | Magic::Mp4 => Some("video"),
Magic::Epub | Magic::PalmDoc | Magic::Pdf => Some("document"),
Magic::Rar4 | Magic::Rar5 | Magic::SevenZip | Magic::Zip => Some("archive"),
// Parity is not a fileType — it is not payload at all. Callers route
// on `is_parity()` before they ever ask for a type.
Magic::Par2 => None,
Magic::Riff | Magic::Unknown => None,
}
}
/// The extension these bytes imply, for naming a materialised file whose
/// posted name carried none (or carried an obfuscated one) — see
/// the usenet plugin's `naming`.
///
/// Abstains wherever the bytes do not pick a single extension, on the same
/// discipline as [`Magic::file_type`]: [`Magic::Riff`] is AVI *and* WAV, and
/// [`Magic::PalmDoc`] is `.prc`/`.mobi`/`.pdb`. [`Magic::Par2`] abstains for
/// a stronger reason — parity is never the payload, so it never gets named.
pub fn extension(self) -> Option<&'static str> {
match self {
Magic::Matroska => Some("mkv"),
Magic::Mp4 => Some("mp4"),
Magic::Pdf => Some("pdf"),
Magic::Epub => Some("epub"),
Magic::Rar4 | Magic::Rar5 => Some("rar"),
Magic::SevenZip => Some("7z"),
Magic::Zip => Some("zip"),
Magic::Riff | Magic::PalmDoc | Magic::Par2 | Magic::Unknown => None,
}
}
/// par2 recovery data: present to repair the payload, never to *be* it.
///
/// One posting on watch.nsl.sh (`…gk2iwj5i…`, claiming `video`/`movie`)
/// materialised to a single 122 MB par2 file and published a dag-pb root
/// over it, because `subject_is_par2` reads the `.par2` extension the
/// obfuscated subject did not carry.
pub fn is_parity(self) -> bool {
matches!(self, Magic::Par2)
}
/// A multi-volume wrapper that must be unwrapped before anything inside it
/// can be addressed. `Epub`/`PalmDoc` are containers in the file-format
/// sense but **not** in this one — they are the document.
pub fn is_archive(self) -> bool {
matches!(self, Magic::Rar4 | Magic::Rar5 | Magic::SevenZip | Magic::Zip)
}
}
/// Bytes needed for a confident answer. The EPUB probe is the deepest reader:
/// local header (30) + `"mimetype"` (8) + extra field + `"application/epub+zip"`
/// (20). 128 covers any sane extra-field length.
pub const SNIFF_LEN: usize = 128;
/// Classify a file from its leading bytes. Short input is fine — every probe
/// bounds-checks and falls through to [`Magic::Unknown`].
pub fn sniff(head: &[u8]) -> Magic {
// RAR5 before RAR4: the RAR4 signature is a prefix of neither, but the
// shared first 6 bytes make ordering worth being explicit about.
if head.starts_with(b"Rar!\x1a\x07\x01\x00") {
return Magic::Rar5;
}
if head.starts_with(b"Rar!\x1a\x07\x00") {
return Magic::Rar4;
}
if head.starts_with(b"PAR2\x00PKT") {
return Magic::Par2;
}
if head.starts_with(b"7z\xbc\xaf\x27\x1c") {
return Magic::SevenZip;
}
if head.starts_with(b"%PDF-") {
return Magic::Pdf;
}
// EBML — Matroska and WebM share it; both demux the same way here.
if head.starts_with(b"\x1a\x45\xdf\xa3") {
return Magic::Matroska;
}
if head.len() >= 8 && &head[4..8] == b"ftyp" {
return Magic::Mp4;
}
if head.starts_with(b"RIFF") {
return Magic::Riff;
}
if head.starts_with(b"PK\x03\x04") {
return if is_epub(head) { Magic::Epub } else { Magic::Zip };
}
if is_palm_ebook(head) {
return Magic::PalmDoc;
}
Magic::Unknown
}
/// An EPUB is a zip whose **first** entry is an uncompressed `mimetype` member
/// holding exactly `application/epub+zip` (OCF 3, §4.2 — the rule exists so the
/// type is readable without inflating anything). That is what separates a book
/// from a `.cbz` or a scene `.zip`.
fn is_epub(head: &[u8]) -> bool {
const NAME_AT: usize = 30;
const NAME: &[u8] = b"mimetype";
const VALUE: &[u8] = b"application/epub+zip";
// Stored (method 0) is mandatory for this member; anything else is not an
// OCF container even if it happens to hold the string.
if head.len() < NAME_AT + NAME.len() || u16::from_le_bytes([head[8], head[9]]) != 0 {
return false;
}
if &head[NAME_AT..NAME_AT + NAME.len()] != NAME {
return false;
}
let extra = u16::from_le_bytes([head[28], head[29]]) as usize;
let at = NAME_AT + NAME.len() + extra;
head.len() >= at + VALUE.len() && &head[at..at + VALUE.len()] == VALUE
}
/// A Palm database (`name[32]`, then `type` at 60 and `creator` at 64) whose
/// type marks it an ebook. Covers MobiPocket (`BOOK`/`MOBI`), eReader
/// (`BOOK`/`PNRd`) and PalmDOC (`TEXt`/`REAd`).
fn is_palm_ebook(head: &[u8]) -> bool {
const TYPE_AT: usize = 60;
head.len() >= TYPE_AT + 4 && matches!(&head[TYPE_AT..TYPE_AT + 4], b"BOOK" | b"TEXt")
}
/// The `fileType` an extension implies, for the agreement check. Mirrors the
/// table in `METADATA_KEYS.md` §`fileType`, whose own source of truth is
/// `filename-tool`'s `ExtentionsMapping.ts`. Only the values reachable from a
/// Usenet posting are listed; anything else answers `None` ("no opinion"),
/// which agrees with everything.
pub fn file_type_from_extension(name: &str) -> Option<&'static str> {
let ext = name.rsplit_once('.')?.1.to_ascii_lowercase();
Some(match ext.as_str() {
"mkv" | "mp4" | "webm" | "avi" | "mov" | "wmv" | "flv" | "m4v" | "mpg" | "mpeg" | "3gp"
| "ts" | "m2ts" | "mts" | "vob" | "ogm" | "ogv" | "divx" | "xvid" => "video",
"mp3" | "wav" | "flac" | "aac" | "ogg" | "oga" | "opus" | "m4a" | "m4b" | "wma" | "mka" => {
"audio"
}
"pdf" | "epub" | "mobi" | "azw" | "azw3" | "djvu" | "txt" | "doc" | "docx" | "prc" => {
"document"
}
"zip" | "rar" | "7z" | "tar" | "gz" | "bz2" | "xz" | "zst" | "cbz" | "cbr" => "archive",
"srt" | "sub" | "ass" | "ssa" | "vtt" | "sbv" | "smi" => "subtitle",
"par2" => return None, // parity, not a fileType — see `Magic::is_parity`
_ => return None,
})
}
/// Resolve one file's `fileType` from its bytes and its name together.
///
/// The registry's rule, verbatim: *"sniff the MIME from the first bytes; if it
/// maps to a definite value, compare it with the extension's value — agreement
/// wins, **disagreement returns `undefined`**"*. `None` here is that
/// `undefined`: it means "these two sources disagree, do not route on it",
/// which is a different and strictly weaker statement than "wrong".
pub fn resolve_file_type(head: &[u8], name: &str) -> Option<&'static str> {
let sniffed = sniff(head).file_type();
let from_name = file_type_from_extension(name);
match (sniffed, from_name) {
(Some(a), Some(b)) if a == b => Some(a),
(Some(_), Some(_)) => None, // disagreement ⇒ undefined
(Some(a), None) => Some(a), // no usable extension: content decides
(None, Some(b)) => Some(b), // unrecognised bytes: the name is all we have
(None, None) => None,
}
}
#[cfg(test)]
#[path = "magic_tests.rs"]
mod tests;
+205
View File
@@ -0,0 +1,205 @@
//! Tests for [`super`] — content sniffing.
//!
//! Every fixture below is **real**: the leading bytes of files materialised on
//! watch.nsl.sh, captured 2026-09-16. Synthetic headers would have agreed with
//! whatever this module happened to implement; these are the bytes that actually
//! defeated the name-based classifiers.
use super::*;
/// Decode a hex fixture. Panics on bad input — these are consts, not user data.
fn hx(s: &str) -> Vec<u8> {
assert!(s.len() % 2 == 0, "odd-length hex fixture");
(0..s.len() / 2)
.map(|i| u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).expect("hex"))
.collect()
}
/// Volume 1 of 35 of the trigger posting — `[smol] Tensei Oujo … S01E05v2`,
/// `bagcsaaa5brqxa2jonz5geltmnftglnrqztt2n23topk2bsnglvhjmty`. Extension-less
/// name, RAR5, method 0 (store). The EBML header of the contained MKV is
/// visible inside these same 200 bytes, at the volume's data offset.
const RAR5_VOL1: &str = "526172211a0701002409274c1001050c010b0101bac0d7af80808080004ce244be8b0102130b94bfd7af80800004eefaa1818d8000b683021a946591800001635b736d6f6c5d2054656e736569204f756a6f20746f2054656e736169205265696a6f75206e6f204d61686f75204b616b756d6569202d205330314530357632202842442031303830702048455643204f70757329205b39394434344436465d2e6d6b760a0313184c91644af1d2231a45dfa3a34286810142f7810142f2810442f381084282886d61";
/// Volume 2 — a *continuation* volume. Byte-identical signature to volume 1,
/// which is why "does it start with `Rar!`" cannot tell you which volume you
/// hold, and why electing "the largest file" picks one at random.
const RAR5_VOL2: &str = "526172211a0701000d6002471101050c03010b0101bac0d7af808080800030554b778b01021b0b93bfd7af80800004eefaa1818d8000b6830236ead743800001635b736d6f6c5d2054656e736569204f756a6f20746f2054656e736169205265696a6f75206e6f204d61686f75204b616b756d6569202d205330314530357632202842442031303830702048455643204f70757329205b39394434344436465d2e6d6b760a0313184c91644af1d223da5b7aaacbc910a46f8b3a8886bc4bfaff0518a54e4a2eb791";
/// RAR4 head, from `…gkigqufwwvhpkbhijvufrdcshogy` (claims `video`/`movie`).
const RAR4: &str = "526172211a0700197a7311000d00000000000000d9427423916f0069ff3f067a";
/// par2 head, from `…gk2iwj5iy3cyzafghckcab7j2fuy` — the posting that
/// materialised to a *single* par2 file and published a dag-pb root over it.
const PAR2: &str = "5041523200504b54083b1a00000000000226e38aa627ac7ee833f4fc92637a69";
/// A genuine EPUB — `Brandon Sanderson - Mistborn`, `…gkdthedplbm5qwmtedo2ymyct2si`.
/// Stored `mimetype` member first, exactly as OCF requires.
const EPUB: &str = "504b0304140000000000ee523a486f61ab2c1400000014000000080000006d696d65747970656170706c69636174696f6e2f657075622b7a6970504b0304140002080800ee523a4827f0a79ab4000000";
/// A MobiPocket Palm database — the `CR!…`-named ebook from `…gle4fanocf5txcdocnto6dh4au4a`.
/// Type `BOOK` at offset 60, creator `MOBI` at 64.
const PALM: &str = "43522158593841474857463731314e4642394a44314443314d374b4548344d000000000057d30d1e57d3113200000000000000000000000000000000424f4f4b4d4f4249000001df";
/// A Matroska head from a posting that resolved correctly.
const MKV: &str = "1a45dfa3a34286810142f7810142f281";
// ---- the six shapes that mattered -------------------------------------------
#[test]
fn the_real_headers_are_each_recognised() {
assert_eq!(sniff(&hx(RAR5_VOL1)), Magic::Rar5);
assert_eq!(sniff(&hx(RAR5_VOL2)), Magic::Rar5);
assert_eq!(sniff(&hx(RAR4)), Magic::Rar4);
assert_eq!(sniff(&hx(PAR2)), Magic::Par2);
assert_eq!(sniff(&hx(EPUB)), Magic::Epub);
assert_eq!(sniff(&hx(PALM)), Magic::PalmDoc);
assert_eq!(sniff(&hx(MKV)), Magic::Matroska);
}
/// The regression that started this: three correct ebook postings on the box are
/// zips, and a "zip ⇒ archive" rule condemns all of them. An EPUB is the
/// document, not a wrapper around one.
#[test]
fn an_epub_is_a_document_not_an_archive() {
let m = sniff(&hx(EPUB));
assert_eq!(m.file_type(), Some("document"));
assert!(!m.is_archive(), "an epub must not route into the unwrapper");
}
/// A `.cbz` and a scene `.zip` share the EPUB signature for four bytes and
/// diverge at the first member. `METADATA_KEYS.md` §`fileType` puts `cbz` in
/// `archive`, so plain-zip ⇒ archive is the correct fallback.
#[test]
fn a_zip_without_the_ocf_mimetype_member_is_an_archive() {
let mut z = hx(EPUB);
// Corrupt just the mimetype *value*; everything else stays a valid zip.
let at = 38;
z[at..at + 3].copy_from_slice(b"XXX");
assert_eq!(sniff(&z), Magic::Zip);
assert_eq!(sniff(&z).file_type(), Some("archive"));
}
/// OCF requires the `mimetype` member to be *stored*. A deflated one is not an
/// OCF container even if the bytes happen to spell the string.
#[test]
fn a_deflated_mimetype_member_is_not_an_epub() {
let mut z = hx(EPUB);
z[8] = 8; // method = deflate
assert_eq!(sniff(&z), Magic::Zip);
}
#[test]
fn parity_is_not_a_file_type_at_all() {
let m = sniff(&hx(PAR2));
assert!(m.is_parity());
assert_eq!(m.file_type(), None, "par2 must never present as payload");
}
#[test]
fn every_rar_generation_reads_as_an_archive() {
for f in [RAR4, RAR5_VOL1, RAR5_VOL2] {
let m = sniff(&hx(f));
assert!(m.is_archive());
assert_eq!(m.file_type(), Some("archive"));
assert!(!m.is_parity());
}
}
// ---- Unknown is an answer, not an error -------------------------------------
#[test]
fn unrecognised_and_truncated_input_is_unknown_not_a_panic() {
assert_eq!(sniff(b""), Magic::Unknown);
assert_eq!(sniff(b"Rar"), Magic::Unknown);
assert_eq!(sniff(b"PK\x03"), Magic::Unknown);
assert_eq!(sniff(&[0u8; 8]), Magic::Unknown);
assert_eq!(sniff(b"not a container at all"), Magic::Unknown);
assert_eq!(sniff(&hx(MKV)[..2]), Magic::Unknown);
}
/// The holed posting on the box (`…gkisayoy…`, 837 MB of zeros) sniffs Unknown.
/// It is already refused upstream by `disk::first_hole`; this only pins that
/// Unknown carries no `fileType` claim either way.
#[test]
fn a_hole_claims_nothing() {
let zeros = vec![0u8; SNIFF_LEN];
assert_eq!(sniff(&zeros), Magic::Unknown);
assert_eq!(sniff(&zeros).file_type(), None);
}
/// RIFF is AVI *and* WAV — a video type and an audio type. Guessing between
/// them is precisely the disagreement case the registry legislates away.
#[test]
fn riff_is_recognised_but_declines_to_name_a_type() {
let m = sniff(b"RIFF\x00\x00\x00\x00AVI LIST");
assert_eq!(m, Magic::Riff);
assert_eq!(m.file_type(), None);
}
// ---- name + bytes together ---------------------------------------------------
#[test]
fn content_decides_when_the_name_says_nothing() {
// The obfuscated case: no extension anywhere.
assert_eq!(
resolve_file_type(&hx(RAR5_VOL1), "IuYf3jBKJOukfUesqNCw0LoUc1Gqxxmilo0ICJJMeddjB6"),
Some("archive")
);
assert_eq!(resolve_file_type(&hx(MKV), "56GqgfEUMp7RAqF94AGp"), Some("video"));
}
#[test]
fn agreement_between_name_and_bytes_wins() {
assert_eq!(resolve_file_type(&hx(MKV), "The.Movie.2019.1080p.mkv"), Some("video"));
assert_eq!(resolve_file_type(&hx(EPUB), "book.epub"), Some("document"));
}
/// `METADATA_KEYS.md` §`fileType`: *"disagreement returns `undefined`"*. A file
/// named `.mkv` whose bytes are a RAR is not a video and is not reliably an
/// archive either — it is a thing we refuse to route on.
#[test]
fn disagreement_returns_undefined_rather_than_guessing() {
assert_eq!(resolve_file_type(&hx(RAR5_VOL1), "episode.mkv"), None);
assert_eq!(resolve_file_type(&hx(MKV), "release.rar"), None);
}
#[test]
fn an_unreadable_header_falls_back_to_the_extension() {
assert_eq!(resolve_file_type(b"\x00\x01\x02\x03", "movie.mkv"), Some("video"));
assert_eq!(resolve_file_type(b"", "chapter.cbz"), Some("archive"));
}
#[test]
fn neither_source_has_an_opinion() {
assert_eq!(resolve_file_type(b"\x00\x01\x02\x03", "blob"), None);
// par2 is deliberately absent from the extension table: it is parity, and
// `Magic::is_parity` is how callers route it.
assert_eq!(file_type_from_extension("recovery.par2"), None);
}
#[test]
fn the_extension_table_matches_the_registry_for_the_shapes_we_see() {
assert_eq!(file_type_from_extension("a.mkv"), Some("video"));
assert_eq!(file_type_from_extension("a.MKV"), Some("video"));
assert_eq!(file_type_from_extension("a.epub"), Some("document"));
assert_eq!(file_type_from_extension("a.cbz"), Some("archive"));
assert_eq!(file_type_from_extension("a.rar"), Some("archive"));
assert_eq!(file_type_from_extension("a.srt"), Some("subtitle"));
assert_eq!(file_type_from_extension("no-extension-here"), None);
}
/// The MKV inside the trigger posting is *stored*, so its EBML header sits in
/// plaintext a few hundred bytes into volume 1. That is what makes the
/// offset-map unwrap possible at all, and it is worth pinning independently of
/// the RAR parser.
#[test]
fn the_stored_mkv_header_is_visible_inside_volume_one() {
let v1 = hx(RAR5_VOL1);
let ebml = v1
.windows(4)
.position(|w| w == [0x1a, 0x45, 0xdf, 0xa3])
.expect("EBML header should be plainly visible inside a stored RAR volume");
assert_eq!(ebml, 174, "volume 1's data offset, per the RAR5 header chain");
assert_eq!(sniff(&v1[ebml..]), Magic::Matroska);
}
+63
View File
@@ -0,0 +1,63 @@
//! **Transport plugins** — the contract between meta-share's protocol-agnostic
//! hull and the processes that actually move bytes (`meta-transport-torrent`,
//! `meta-transport-nzb`, `meta-transport-ipfs`, or a competing implementation
//! of any of them).
//!
//! Same split as gateway ⇄ feeders: the plugin finds and moves bytes for one
//! protocol; the hull keeps everything that is policy and bookkeeping — the
//! public HTTP API, the material index, seed rows, eviction, focus, meta-core.
//! Plugins never call each other and never talk to meta-core's record API;
//! every cross-tier edge goes through the hull.
//!
//! - [`plugin::TransportPlugin`] + [`serve::serve_transport`]: the plugin side.
//! - [`client::RemoteTransport`]: the hull side.
//! - [`hull::HullClient`]: plugin → hull callbacks (internal port only).
//! - [`config::ConfigPlane`]: the plugin's own settings + the config page the
//! hull's dashboard embeds.
//! - [`focus::FocusView`]: the plugin's replica of the hull's playback focus.
//! - [`dto`]: common wire types; [`error::ApiError`] / [`range`]: shared so a
//! relayed plugin response is byte-identical to what the monolith answered.
//! - [`cid`]: CID parsing + cid-shape decoders, shared so every process
//! classifies a cid the same way.
//! - [`torrent`]: the torrent plugin's extra routes + magnet helpers.
//! - [`ipfs`]: the ipfs plugin's extra routes (the hull's blockstore/swarm facade).
//! - [`nzb`]: Usenet cid shapes, the usenet plugin's extra routes + settings.
//! - [`pointer`]: which cids are pointers (resolved before their bytes exist).
//! - [`magic`]: content sniffing; [`file`]: range-serving a file off `/data`.
//! - `testkit` (feature `testkit`): conformance checks for any implementation.
//!
//! Bytes on disk: every plugin mounts the same `/data` volume as the hull, so a
//! plugin-written file is opened by the hull directly (local-first reads) and
//! `rename(2)` between `tmp/` and `cache/` stays atomic.
pub mod cid;
pub mod client;
pub mod config;
pub mod dto;
pub mod error;
pub mod file;
pub mod focus;
pub mod fsutil;
pub mod hull;
pub mod ipfs;
pub mod magic;
pub mod nzb;
pub mod plugin;
pub mod pointer;
pub mod range;
pub mod serve;
pub mod torrent;
#[cfg(any(test, feature = "testkit"))]
pub mod testkit;
pub use client::RemoteTransport;
pub use config::ConfigPlane;
pub use dto::{
Capabilities, Deleted, Event, Health, Job, Jobs, Manifest, PromotedFile, ReconcileReport,
ReconcileRequest, CONTRACT_VERSION, MIN_CONTRACT,
};
pub use error::ApiError;
pub use focus::{FocusSnapshot, FocusTitle, FocusView, Lane};
pub use hull::{HullClient, NetworkInfo, RecordInfo};
pub use plugin::TransportPlugin;
pub use serve::serve_transport;
+560
View File
@@ -0,0 +1,560 @@
//! Usenet cid shapes, shared by the hull and the usenet transport plugin, plus
//! the plugin's extra-route DTOs and its settings.
//!
//! The codec helpers are moved verbatim from meta-share `nzb/mod.rs`; the hull
//! dispatches and grades with them, the plugin decodes with them, so they must
//! be one copy.
//!
//! Extra routes of the usenet plugin (all under `/nzb/`):
//!
//! - `GET /nzb/status/:cid` → the `kind:"nzb"` status JSON (never starts a job)
//! - `GET /nzb/jobs` → `{"jobs":[…]}` in-flight jobs (`NzbActivity` JSON)
//! - `DELETE /nzb/jobs/:cid` → `{"cancelled":bool}`
//! - `GET /nzb/capability` → [`Capability`]
//! - `GET /nzb/verdict/:cid` → `Option<String>` remembered verdict
//! - `POST /nzb/verify/:cid` → `Option<String>` verdict, verified if free
//! - `POST /nzb/test-nntp` ← [`NntpTest`] → `{"ok","message"}`
//! - `POST /nzb/restart` → `204`, then the process exits (settings changed)
//! - `POST /nzb/seal/:cid/claim` → [`SealClaim`] / `409` (not `Ready`, or already claimed)
//! - `POST /nzb/seal/:cid/promote`→ [`SealPrepared`] (publish into `cache/` + the hole/magic gates)
//! - `POST /nzb/seal/:cid/done` ← [`SealDone`]
//! - `POST /nzb/seal/:cid/abort` → `204`
//!
//! `/raw/:cid` may answer `409` with header [`HDR_REDIRECT_CID`] — "these bytes
//! now live under that cid (an IPFS root) and my copy is gone": the hull
//! re-dispatches to it, exactly where the monolith fell back to the blockstore.
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use super::cid::MsCid;
/// `409` + this header on `/raw`: re-dispatch the request to this cid.
pub const HDR_REDIRECT_CID: &str = "x-metamesh-redirect-cid";
/// `GET /nzb/capability`.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct Capability {
/// The plugin holds an NNTP pool (host+user+pass configured).
pub nntp: bool,
/// The NNTP provider host (never the account), when credentialed.
#[serde(default)]
pub provider: Option<String>,
}
/// `POST /nzb/test-nntp` — a candidate connection, already merged with the
/// stored secret by the hull (which owns `settings.json`).
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct NntpTest {
pub host: String,
pub port: u16,
pub user: String,
pub pass: String,
pub tls: bool,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct SealClaim {
pub path: String,
pub size: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct SealPrepared {
/// The published file, under `cache/<cid>/`.
pub path: String,
/// Its name inside the container.
pub rel: String,
pub size: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct SealDone {
pub root: String,
pub size: u64,
pub path: String,
}
/// Usenet (NZB) retrieval settings — the `usenet` section of the hull's
/// `settings.json`. The plugin reads the file read-only at boot (the hull owns
/// it and restarts the plugin when it changes).
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default)]
pub struct UsenetSettings {
pub nntp_host: Option<String>,
pub nntp_port: Option<u16>,
pub nntp_user: Option<String>,
/// Secret.
pub nntp_pass: Option<String>,
pub nntp_tls: bool,
pub nntp_connections: u32,
}
impl Default for UsenetSettings {
fn default() -> Self {
Self {
nntp_host: None,
nntp_port: None,
nntp_user: None,
nntp_pass: None,
nntp_tls: true,
nntp_connections: 20,
}
}
}
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key)
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
}
impl UsenetSettings {
/// DEPRECATED seed path: the legacy `META_SHARE_NNTP_*` env vars. Only
/// consulted when `settings.json` doesn't exist yet.
pub fn from_env() -> Self {
let mut s = Self::default();
if let Some(v) = env_nonempty("META_SHARE_NNTP_HOST") {
s.nntp_host = Some(v);
}
if let Some(v) = env_nonempty("META_SHARE_NNTP_PORT").and_then(|v| v.parse().ok()) {
s.nntp_port = Some(v);
}
if let Some(v) = env_nonempty("META_SHARE_NNTP_USER") {
s.nntp_user = Some(v);
}
if let Some(v) = env_nonempty("META_SHARE_NNTP_PASS") {
s.nntp_pass = Some(v);
}
if let Ok(v) = std::env::var("META_SHARE_NNTP_TLS") {
s.nntp_tls = !matches!(v.trim(), "0" | "false" | "no");
}
if let Some(v) = env_nonempty("META_SHARE_NNTP_CONNECTIONS").and_then(|v| v.parse().ok()) {
s.nntp_connections = v;
}
s.nntp_connections = s.nntp_connections.clamp(1, 100);
s
}
/// The `usenet` section of `<config_dir>/settings.json`, or the legacy env
/// seed when the file is absent or unparseable.
pub fn load(config_dir: &std::path::Path) -> Self {
#[derive(Deserialize, Default)]
#[serde(default)]
struct File {
usenet: UsenetSettings,
}
match std::fs::read(config_dir.join("settings.json")) {
Ok(bytes) => match serde_json::from_slice::<File>(&bytes) {
Ok(f) => f.usenet,
Err(_) => Self::from_env(),
},
Err(_) => Self::from_env(),
}
}
}
/// MetaMesh-private multicodec for a self-describing Newznab **release** locator
/// (`nzb-release`). Mirrors `meta-feeder-sdk`'s `hash::NZB_RELEASE_CODEC`. The
/// cid's identity multihash embeds `{host, id}` directly, so the feeder emits it
/// with **no `.nzb` download** and the credentialed peer decodes the locator
/// straight from the cid at playback — no KV side-table.
pub const NZB_RELEASE_CODEC: u64 = 0x1005;
/// MetaMesh-private multicodec for a **self-scanned Usenet posting**
/// (`nzb-posting`). Mirrors `meta-feeder-sdk`'s `hash::NZB_POSTING_CODEC`.
///
/// The crucial difference from [`NZB_RELEASE_CODEC`]: this is a **digest over
/// the article Message-ID set**, not a locator. It embeds no indexer host, so
/// **any peer with a plain NNTP provider can redeem it — no indexer credential**.
/// That portability is the entire point of scanning Usenet ourselves.
///
/// Two consequences the code must respect:
/// - It is **not reversible**. There is no `decode_*` for it. The `.nzb`
/// manifest arrives out of band, as an ordinary sha2-256 cid named by the
/// record's `manifest` field (see the hull's manifest resolution).
/// - It is **not bitswap-fetchable under itself**. Bitswap derives a block's
/// cid by hashing the block; this digest is over the id set, so a want for it
/// could never be satisfied. Never offer it to the swarm.
///
/// See `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5.
pub const NZB_POSTING_CODEC: u64 = 0x1003;
/// Does this cid address a Usenet **release locator** (codec `0x1005`) — the
/// self-describing `{host, id}` form that must be redeemed against a specific
/// indexer with a matching credential?
///
/// Narrower than [`is_nzb_cid`] on purpose: only the locator carries a host, so
/// only the locator gates on an indexer key (held by a gateway's feeder). Use this where
/// the *host* matters; use [`is_nzb_cid`] where "should this go to the Usenet
/// transport" matters.
pub fn is_nzb_locator(mscid: &MsCid) -> bool {
mscid.codec() == NZB_RELEASE_CODEC
}
/// Does this cid address a **self-scanned Usenet posting** (codec `0x1003`)?
pub fn is_nzb_posting(mscid: &MsCid) -> bool {
mscid.codec() == NZB_POSTING_CODEC
}
/// Does this cid go to the **Usenet transport** at all — either form?
///
/// This is the `/raw` dispatch predicate: both shapes end at
/// the usenet plugin's `/raw`, which resolves a manifest (differently per
/// form — see the hull's manifest resolution), NNTP-fetches the articles, and
/// materialises/streams the media. Everything downstream of manifest
/// resolution is identical for the two.
pub fn is_nzb_cid(mscid: &MsCid) -> bool {
is_nzb_locator(mscid) || is_nzb_posting(mscid)
}
/// String convenience over [`is_nzb_cid`] — either Usenet form. Used by the
/// playback-focus endpoints, which see the bare CID string. There is no
/// locator-only counterpart: every string-typed call site that used to ask
/// "is this specifically a `0x1005` locator" now needs "is this Usenet at
/// all" — see the six dispatch sites widened alongside `nzb-posting`.
pub fn cid_is_nzb(cid: &str) -> bool {
super::cid::parse_record_cid(cid)
.map(|c| is_nzb_cid(&c))
.unwrap_or(false)
}
/// Re-derive an `nzb-posting` digest from a manifest's Message-IDs.
///
/// ⚠ **MIRRORED LOGIC** — byte-for-byte equivalent to
/// `meta_feeder_sdk::hash::compute_nzb_posting_cid` (the gateway/feeder side
/// mints with it; we verify with it). The two crates cannot share a path-dep,
/// so the normalisation rule is duplicated and must be changed on both sides
/// together. A drift here does not error — it silently makes every self-scanned
/// posting fail verification and become unplayable.
///
/// The rule: trim, strip a surrounding `<…>`, drop empties, dedupe, sort
/// lexicographically, join with `\n`, SHA-256. Case is preserved (a Message-ID
/// local part is case-sensitive per RFC 5322).
pub fn nzb_posting_digest<S: AsRef<str>>(message_ids: &[S]) -> [u8; 32] {
use sha2::{Digest, Sha256};
let mut ids: Vec<&str> = message_ids
.iter()
.map(|s| {
let t = s.as_ref().trim();
t.strip_prefix('<')
.and_then(|r| r.strip_suffix('>'))
.unwrap_or(t)
})
.filter(|s| !s.is_empty())
.collect();
ids.sort_unstable();
ids.dedup();
let mut hasher = Sha256::new();
for (i, id) in ids.iter().enumerate() {
if i > 0 {
hasher.update(b"\n");
}
hasher.update(id.as_bytes());
}
hasher.finalize().into()
}
/// A decoded self-describing `nzb-release` locator: the Newznab **API base**,
/// recovered from the cid's identity-multihash digest. The release id stays in
/// the cid — only the feeder that redeems it reads it.
pub struct NzbLocator {
/// The indexer's Newznab API base, **scheme-less, authority + optional
/// path** (`api.nzb.life`, `api.nzbgeek.info/api`). Stamped by the feeder
/// from the indexer aggregator's own configured `baseUrl` — i.e. the URL
/// that aggregator itself queries, so it is redeemable by construction.
///
/// ⚠ This used to be a bare `host` taken from the release's `<guid>`
/// **permalink**, which is only the API host by coincidence. nzb.life
/// serves both from `api.nzb.life` so it worked; nzbgeek's permalinks are on
/// `nzbgeek.info` while its API is on `api.nzbgeek.info/api`, so every
/// nzbgeek release was unredeemable — `https://nzbgeek.info/api?t=get`
/// answers 302→HTML and the grab died in `nzb xml parse`.
///
/// Because it may carry a path it is **not** a matching key: use
/// [`authority()`](Self::authority) for that.
pub api_base: String,
}
impl NzbLocator {
/// The host to match against the gateways' redeem claims — the authority
/// alone, path stripped.
///
/// The base may carry a path (`api.nzbgeek.info/api`) that a claim never
/// does: operators configure, and feeders advertise, plain hosts.
pub fn authority(&self) -> &str {
match self.api_base.split_once('/') {
Some((authority, _path)) => authority,
None => &self.api_base,
}
}
}
/// Normalise an indexer host for matching: drop any `http(s)://` scheme, trim a
/// trailing slash and surrounding whitespace, lowercase. A locator's
/// [`NzbLocator::authority`] passes through this before it is compared with the
/// hosts a gateway advertises, so a claim for `https://api.example.com/` and one
/// for `api.example.com` match the same releases.
pub fn normalize_indexer_host(raw: &str) -> String {
raw.trim()
.trim_start_matches("https://")
.trim_start_matches("http://")
.trim_end_matches('/')
.to_ascii_lowercase()
}
/// Decode a self-describing `nzb-release` cid (codec `0x1005`) into its
/// API base. The identity-multihash digest is
/// `varint(base_len) ‖ api_base ‖ id_bytes` — the exact inverse of
/// meta-feeder-sdk's `hash::compute_nzb_release_cid`; `id_bytes` is the
/// hex-decoded release id, which only the redeeming feeder reads. Errors
/// (mapped to 4xx by the caller) on a wrong codec or a malformed digest.
///
/// The wire shape is unchanged from when this field held a bare host — only the
/// *meaning* of those bytes widened to "authority + optional path", so old cids
/// still decode (they just name an unredeemable host; see [`NzbLocator`]).
pub fn decode_nzb_release_cid(mscid: &MsCid) -> Result<NzbLocator> {
anyhow::ensure!(
mscid.codec() == NZB_RELEASE_CODEC,
"not an nzb-release cid (codec 0x{:x}, want 0x{:x})",
mscid.codec(),
NZB_RELEASE_CODEC
);
let digest = mscid.hash().digest();
let (base_len, consumed) =
super::cid::read_uvarint(digest).context("nzb-release digest: base-length varint")?;
let base_len = usize::try_from(base_len)
.map_err(|_| anyhow::anyhow!("nzb-release base length {base_len} overflows usize"))?;
let rest = &digest[consumed..];
anyhow::ensure!(
rest.len() > base_len,
"nzb-release digest truncated: base_len={base_len}, only {} byte(s) left (need base + id)",
rest.len()
);
let base_bytes = &rest[..base_len];
let api_base = std::str::from_utf8(base_bytes)
.context("nzb-release api base is not valid utf-8")?
.to_string();
Ok(NzbLocator { api_base })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::transport::cid::parse_record_cid;
#[test]
fn settings_load_reads_the_usenet_section_and_ignores_the_rest() {
let d = std::env::temp_dir().join(format!("nzb-settings-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
std::fs::write(
d.join("settings.json"),
r#"{"network":{"meta_core_url":"x"},"usenet":{"nntp_host":"news.example","nntp_user":"u","nntp_pass":"p","nntp_tls":true,"nntp_connections":7,"indexers":[1]}}"#,
)
.unwrap();
let s = UsenetSettings::load(&d);
assert_eq!(s.nntp_host.as_deref(), Some("news.example"));
assert_eq!(s.nntp_connections, 7);
let _ = std::fs::remove_dir_all(&d);
}
/// RFC 4648 base32 lowercase, no padding (multibase `b` body).
fn base32_lower(input: &[u8]) -> String {
const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
let mut out = String::new();
let (mut buffer, mut bits): (u64, u32) = (0, 0);
for &b in input {
buffer = (buffer << 8) | b as u64;
bits += 8;
while bits >= 5 {
bits -= 5;
out.push(ALPHABET[((buffer >> bits) & 0x1F) as usize] as char);
}
}
if bits > 0 {
out.push(ALPHABET[((buffer << (5 - bits)) & 0x1F) as usize] as char);
}
out
}
fn varint(mut v: u64, out: &mut Vec<u8>) {
while v >= 0x80 {
out.push((v as u8) | 0x80);
v >>= 7;
}
out.push(v as u8);
}
/// Build a self-describing nzb-release cid exactly as meta-feeder-sdk's
/// `compute_nzb_release_cid` does: identity multihash over
/// `varint(host_len) ‖ host ‖ id_bytes`.
fn make_nzb_cid(host: &str, id_bytes: &[u8]) -> String {
let mut digest = Vec::new();
varint(host.len() as u64, &mut digest);
digest.extend_from_slice(host.as_bytes());
digest.extend_from_slice(id_bytes);
let mut wire = vec![0x01u8];
varint(NZB_RELEASE_CODEC, &mut wire);
wire.push(0x00); // identity multihash
varint(digest.len() as u64, &mut wire);
wire.extend_from_slice(&digest);
format!("b{}", base32_lower(&wire))
}
#[test]
fn decode_round_trips_the_base() {
let cid = make_nzb_cid("api.example.com", &[0x48, 0x5c, 0x52, 0xf0]);
let mscid = parse_record_cid(&cid).expect("parse");
assert!(is_nzb_locator(&mscid));
let loc = decode_nzb_release_cid(&mscid).expect("decode");
assert_eq!(loc.api_base, "api.example.com");
// No path ⇒ authority is the whole thing.
assert_eq!(loc.authority(), "api.example.com");
}
/// The nzbgeek shape, and the whole reason `authority()` exists: its API
/// lives at `https://api.nzbgeek.info/api` while a feeder claims the plain
/// host. The base keeps the path; every match must drop it.
#[test]
fn a_base_with_a_path_matches_on_its_authority() {
let cid = make_nzb_cid("api.nzbgeek.info/api", &[0x19, 0xfe, 0x38, 0xe4]);
let mscid = parse_record_cid(&cid).expect("parse");
let loc = decode_nzb_release_cid(&mscid).expect("decode");
assert_eq!(loc.api_base, "api.nzbgeek.info/api");
assert_eq!(
loc.authority(),
"api.nzbgeek.info",
"can_redeem / probe / redeem claims all compare on the authority"
);
}
/// A port is part of the authority, not a path — it must survive the split.
#[test]
fn a_port_stays_in_the_authority() {
let cid = make_nzb_cid("indexer.local:5076/api", &[0x01, 0x02]);
let mscid = parse_record_cid(&cid).expect("parse");
let loc = decode_nzb_release_cid(&mscid).expect("decode");
assert_eq!(loc.authority(), "indexer.local:5076");
}
#[test]
fn decode_rejects_wrong_codec() {
// codec 0x1001 (btih-v1-file), not an nzb-release locator.
let mut wire = vec![0x01u8];
varint(0x1001, &mut wire);
wire.extend_from_slice(&[0x00, 0x02, 0xAB, 0xCD]); // identity mh, 2-byte digest
let cid = format!("b{}", base32_lower(&wire));
let mscid = parse_record_cid(&cid).expect("parse");
assert!(!is_nzb_locator(&mscid));
assert!(decode_nzb_release_cid(&mscid).is_err());
}
#[test]
fn decode_rejects_truncated_digest() {
// host_len claims the whole digest, leaving no id bytes.
let cid = make_nzb_cid("api.example.com", &[]);
let mscid = parse_record_cid(&cid).expect("parse");
assert!(decode_nzb_release_cid(&mscid).is_err());
}
/// Build an nzb-posting cid exactly as meta-feeder-sdk's
/// `compute_nzb_posting_cid` does: codec 0x1003 + sha2-256 over the
/// normalised, sorted, newline-joined Message-ID set.
fn make_posting_cid(message_ids: &[&str]) -> String {
let digest = nzb_posting_digest(message_ids);
let mut wire = vec![0x01u8];
varint(NZB_POSTING_CODEC, &mut wire);
wire.push(0x12); // sha2-256
wire.push(0x20); // 32 bytes
wire.extend_from_slice(&digest);
format!("b{}", base32_lower(&wire))
}
/// ⚠ THE MIRROR TEST. `nzb_posting_digest` here must agree byte-for-byte
/// with `meta_feeder_sdk::hash::compute_nzb_posting_cid`'s digest — the
/// gateway mints with one, we verify with the other, and a drift silently
/// makes every self-scanned posting unplayable rather than erroring.
///
/// This vector is the same one pinned on the SDK side
/// (`hash::tests::nzb_posting_cid_wire_shape`): a single id `[email protected]`
/// digests to plain `sha256("[email protected]")`.
#[test]
fn posting_digest_matches_the_sdk_mint() {
use sha2::{Digest, Sha256};
let expect: [u8; 32] = Sha256::digest(b"[email protected]").into();
assert_eq!(nzb_posting_digest(&["[email protected]"]), expect);
}
/// The normalisation rule, mirrored from the SDK: order-independent,
/// bracket/blank/dupe-insensitive, case-SENSITIVE.
#[test]
fn posting_digest_normalisation_matches_the_mint() {
// Order does not matter — this is why we hash a sorted set and never
// the raw `.nzb` bytes (the retired 0x1004's mistake).
assert_eq!(
nzb_posting_digest(&["[email protected]", "[email protected]", "[email protected]"]),
nzb_posting_digest(&["[email protected]", "[email protected]", "[email protected]"])
);
// Angle brackets, whitespace, empties and duplicates all normalise away.
assert_eq!(
nzb_posting_digest(&["[email protected]", "[email protected]"]),
nzb_posting_digest(&[" <[email protected]> ", "[email protected]", "", " ", "[email protected]"])
);
// Case is preserved (RFC 5322: the local part is case-sensitive).
assert_ne!(
nzb_posting_digest(&["[email protected]"]),
nzb_posting_digest(&["[email protected]"])
);
// A different article set is a different posting.
assert_ne!(
nzb_posting_digest(&["[email protected]", "[email protected]"]),
nzb_posting_digest(&["[email protected]", "[email protected]"])
);
}
/// Both Usenet forms route to the Usenet transport, but only the `0x1005`
/// locator is host-bound. Keeping the two predicates distinct is what lets
/// `can_redeem` skip the credential check for a posting.
#[test]
fn posting_and_locator_are_both_nzb_but_only_one_is_a_locator() {
let posting = parse_record_cid(&make_posting_cid(&["[email protected]"])).expect("parse posting");
assert!(is_nzb_posting(&posting));
assert!(is_nzb_cid(&posting), "postings go to the Usenet transport");
assert!(
!is_nzb_locator(&posting),
"a posting names no indexer host, so it must NOT be treated as a locator (that would send it down the credential-gated grab path)"
);
let locator = parse_record_cid(&make_nzb_cid("api.example.com", &[0xab, 0xcd]))
.expect("parse locator");
assert!(is_nzb_locator(&locator));
assert!(is_nzb_cid(&locator));
assert!(!is_nzb_posting(&locator));
}
/// A posting cid must never decode as a release locator — the digest is not
/// a `{host, id}` payload, and misreading it would produce a garbage host.
#[test]
fn posting_cid_does_not_decode_as_a_locator() {
let posting = parse_record_cid(&make_posting_cid(&["[email protected]", "[email protected]"]))
.expect("parse posting");
assert!(decode_nzb_release_cid(&posting).is_err());
}
/// The string convenience wrappers agree with the parsed forms, including
/// on garbage input (which must be `false`, never a panic).
#[test]
fn cid_string_helpers_agree() {
let posting = make_posting_cid(&["[email protected]"]);
assert!(cid_is_nzb(&posting));
let locator = make_nzb_cid("api.example.com", &[0x01]);
assert!(cid_is_nzb(&locator));
assert!(!cid_is_nzb("not-a-cid"));
}
}
+58
View File
@@ -0,0 +1,58 @@
//! The plugin side of the contract: what a transport implementation provides.
use std::sync::Arc;
use async_trait::async_trait;
use axum::http::HeaderMap;
use axum::response::Response;
use axum::Router;
use super::config::ConfigPlane;
use super::dto::{Deleted, Health, Job, Manifest, ReconcileReport, ReconcileRequest};
use super::error::ApiError;
use super::focus::FocusView;
/// One transport implementation (BitTorrent, Usenet, IPFS, or a competing
/// implementation of one of them). [`serve_transport`](super::serve::serve_transport)
/// wraps it in the common routes; [`extra_routes`](Self::extra_routes) adds its
/// protocol-specific ones.
#[async_trait]
pub trait TransportPlugin: Send + Sync + 'static {
fn manifest(&self) -> Manifest;
async fn health(&self) -> Health {
Health::ok()
}
/// `GET /raw/:cid`. Must answer exactly what the in-process tier answered
/// in the monolith (status, `Content-*`, `Retry-After`, error body) — the
/// hull relays it to meta-share's public API untouched. Must **not** meter
/// its own egress: the hull does, on relay.
async fn raw(&self, cid: String, headers: HeaderMap) -> Response;
/// `GET /jobs`.
async fn jobs(&self) -> Result<Vec<Job>, ApiError>;
/// `DELETE /jobs/:cid?keep=…`. `keep` lists sibling cids of the same unit
/// the hull still wants.
async fn delete(&self, cid: String, keep: Vec<String>) -> Result<Deleted, ApiError>;
/// `POST /reconcile`.
async fn reconcile(&self, req: ReconcileRequest) -> Result<ReconcileReport, ApiError>;
/// The view `PUT /focus` writes into.
fn focus(&self) -> &Arc<FocusView>;
/// The plugin's own settings, if it has any. `Some` mounts the `/config*`
/// routes and sets `manifest.config` — the hull's dashboard then offers a
/// "configure" panel that embeds `/config` through its proxy.
fn config(&self) -> Option<Arc<ConfigPlane>> {
None
}
/// Protocol-specific routes, merged into the common router. Paths must not
/// collide with the common ones.
fn extra_routes(self: Arc<Self>) -> Router {
Router::new()
}
}
+248
View File
@@ -0,0 +1,248 @@
//! **Pointer cids** — cids that name something to fetch rather than hash it.
//!
//! A pointer's multihash is `identity`: the cid *contains* the address (an
//! indexer host + release id, a provider source + file id), not a digest of the
//! bytes. Bitswap cannot answer one, because there is no hash to verify against.
//! It is resolved instead: by a manifest this peer already stored (the record's
//! pointer field names its content cid), or by a **redeem** at a gateway whose
//! feeder holds the credential for `key`.
//!
//! This module is the single place that knows which codecs are pointers, so the
//! hull ("resolve it first") and the ipfs plugin ("here is how") agree.
//!
//! | codec | what it points at | redeem key | record field |
//! |---|---|---|---|
//! | `0x1005` nzb-release | a `.nzb` at an indexer | indexer host | `manifest` |
//! | `0x1003` nzb-posting | a self-scanned `.nzb` | — (record only) | `manifest` |
//! | `0x100A` provider-file | a file a provider holds | provider source | `file` |
use anyhow::{Context, Result};
use super::cid::{parse_record_cid, read_uvarint};
use super::nzb::{
decode_nzb_release_cid, normalize_indexer_host, NZB_POSTING_CODEC, NZB_RELEASE_CODEC,
};
/// MetaMesh-private multicodec for a provider-held file (`docs/cid-formats.md` §8).
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
/// Redeem-claim codec names, as gateways advertise them in
/// `GET /api/gateway/plugins` (`redeems[].codec`).
pub const REDEEM_NZB_RELEASE: &str = "nzb-release";
pub const REDEEM_PROVIDER_FILE: &str = "provider-file";
/// Record fields a redeem merges onto the pointer's record: the content cid of
/// what it resolved to.
pub const FIELD_MANIFEST: &str = "manifest";
pub const FIELD_FILE: &str = "file";
/// Identity multihash code: the digest bytes are the payload verbatim.
const IDENTITY_MULTIHASH: u64 = 0x00;
/// A decoded pointer.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Pointer {
/// The redeem-claim codec, or `None` when no gateway can redeem it (a
/// self-scanned posting resolves from the record alone).
pub redeem: Option<RedeemKey>,
/// The record field naming the resolved content cid.
pub field: &'static str,
/// What the resolved bytes are: a manifest a byte plugin consumes, or the
/// file itself.
pub kind: PointerKind,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RedeemKey {
/// [`REDEEM_NZB_RELEASE`] or [`REDEEM_PROVIDER_FILE`].
pub codec: &'static str,
/// The indexer host / provider source a gateway's claim must cover.
pub key: String,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PointerKind {
/// A `.nzb`: handed to the nzb plugin, which fetches the bytes.
NzbManifest,
/// The file itself (a subtitle): served as is.
File,
}
/// Decode `cid` as a pointer.
///
/// - `None` — not a pointer: an ordinary content cid, or unparseable.
/// - `Some(Err(_))` — the codec says pointer but the payload is malformed; the
/// caller answers `400`.
pub fn decode(cid: &str) -> Option<Result<Pointer>> {
match codec_of(cid)? {
NZB_RELEASE_CODEC => Some(decode_nzb_release(cid)),
NZB_POSTING_CODEC => Some(Ok(Pointer {
redeem: None,
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})),
PROVIDER_FILE_CODEC => Some(decode_provider_file(cid).map(|(source, _id)| Pointer {
redeem: Some(RedeemKey { codec: REDEEM_PROVIDER_FILE, key: source }),
field: FIELD_FILE,
kind: PointerKind::File,
})),
_ => None,
}
}
/// Is `cid` a pointer (well-formed or not)?
pub fn is_pointer(cid: &str) -> bool {
decode(cid).is_some()
}
fn decode_nzb_release(cid: &str) -> Result<Pointer> {
let mscid = parse_record_cid(cid)?;
let locator = decode_nzb_release_cid(&mscid)?;
Ok(Pointer {
redeem: Some(RedeemKey {
codec: REDEEM_NZB_RELEASE,
key: normalize_indexer_host(locator.authority()),
}),
field: FIELD_MANIFEST,
kind: PointerKind::NzbManifest,
})
}
/// Decode a `provider-file` cid into `(source, id)`. Errors on a different
/// codec or a malformed payload. Works on the raw multibase string because a
/// long provider id can exceed the 64-byte multihash a typed cid parse allows.
pub fn decode_provider_file(cid: &str) -> Result<(String, String)> {
let bytes = raw_cid_bytes(cid).context("provider-file: not a base32 CIDv1")?;
let mut at = 0usize;
let next = |at: &mut usize, what: &str| -> Result<u64> {
let (v, n) = read_uvarint(&bytes[*at..])
.with_context(|| format!("provider-file: {what} varint"))?;
*at += n;
Ok(v)
};
anyhow::ensure!(next(&mut at, "version")? == 1, "provider-file: not a CIDv1");
anyhow::ensure!(next(&mut at, "codec")? == PROVIDER_FILE_CODEC, "not a provider-file cid");
let mh = next(&mut at, "multihash-code")?;
anyhow::ensure!(
mh == IDENTITY_MULTIHASH,
"provider-file locator must use the identity multihash (0x00), got 0x{mh:x}"
);
let len = next(&mut at, "digest-length")?;
anyhow::ensure!(
(bytes.len() - at) as u64 == len,
"provider-file digest length {len} != {} trailing byte(s)",
bytes.len() - at
);
let source_len = next(&mut at, "source-length")?;
let rest = &bytes[at..];
let source_len = usize::try_from(source_len)
.ok()
.filter(|n| *n <= rest.len())
.context("provider-file: source length overruns the digest")?;
let (source, id) = rest.split_at(source_len);
let source = std::str::from_utf8(source).context("provider-file source is not utf-8")?;
let id = std::str::from_utf8(id).context("provider-file id is not utf-8")?;
anyhow::ensure!(
!source.is_empty() && !id.is_empty(),
"provider-file locator needs both a source and an id"
);
Ok((source.to_string(), id.to_string()))
}
/// The codec of a base32 CIDv1 string, without the 64-byte multihash limit.
fn codec_of(cid: &str) -> Option<u64> {
let bytes = raw_cid_bytes(cid)?;
let (version, n) = read_uvarint(&bytes).ok()?;
if version != 1 {
return None;
}
read_uvarint(&bytes[n..]).ok().map(|(codec, _)| codec)
}
/// Decode a `b`-prefixed lowercase base32 (RFC 4648, no padding) CID string.
fn raw_cid_bytes(cid: &str) -> Option<Vec<u8>> {
let body = cid.strip_prefix('b')?;
let mut out = Vec::with_capacity(body.len() * 5 / 8);
let (mut acc, mut bits) = (0u32, 0u32);
for c in body.bytes() {
let v = match c {
b'a'..=b'z' => c - b'a',
b'2'..=b'7' => c - b'2' + 26,
_ => return None,
} as u32;
acc = (acc << 5) | v;
bits += 5;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
Some(out)
}
#[cfg(test)]
mod tests {
use super::*;
/// The Ragna Crimson release from watch.nsl.sh (2026-10-04).
const NZBGEEK: &str = "bagcsaabfcrqxa2jonz5gez3fmvvs42lomzxs6ylqngxuch22nhfhl4v4vmyhiegah2ka";
fn b32(bytes: &[u8]) -> String {
const A: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567";
let (mut acc, mut bits, mut s) = (0u32, 0u32, String::from("b"));
for &b in bytes {
acc = (acc << 8) | b as u32;
bits += 8;
while bits >= 5 {
bits -= 5;
s.push(A[((acc >> bits) & 31) as usize] as char);
}
acc &= (1 << bits) - 1;
}
if bits > 0 {
s.push(A[((acc << (5 - bits)) & 31) as usize] as char);
}
s
}
fn provider_file(source: &str, id: &str) -> String {
let mut digest = vec![source.len() as u8];
digest.extend_from_slice(source.as_bytes());
digest.extend_from_slice(id.as_bytes());
// version 1, codec 0x100A (varint 0x8a 0x20), identity, length
let mut wire = vec![0x01, 0x8a, 0x20, 0x00, digest.len() as u8];
wire.extend_from_slice(&digest);
b32(&wire)
}
#[test]
fn an_nzb_release_is_a_pointer_keyed_by_its_indexer_host() {
let p = decode(NZBGEEK).unwrap().unwrap();
assert_eq!(
p.redeem,
Some(RedeemKey { codec: REDEEM_NZB_RELEASE, key: "api.nzbgeek.info".into() })
);
assert_eq!(p.field, FIELD_MANIFEST);
assert_eq!(p.kind, PointerKind::NzbManifest);
}
#[test]
fn a_provider_file_is_a_pointer_keyed_by_its_source() {
let cid = provider_file("opensubtitles", "file:7061834");
assert_eq!(
decode_provider_file(&cid).unwrap(),
("opensubtitles".into(), "file:7061834".into())
);
let p = decode(&cid).unwrap().unwrap();
assert_eq!(p.redeem.unwrap().key, "opensubtitles");
assert_eq!(p.field, FIELD_FILE);
assert_eq!(p.kind, PointerKind::File);
}
#[test]
fn a_content_cid_is_not_a_pointer() {
assert!(!is_pointer("bafkreigh2akiscaildcqabsyg3dfr6chu3fgpregiymsck7e7aqa4s52zy"));
assert!(!is_pointer("not a cid"));
}
}
+277
View File
@@ -0,0 +1,277 @@
//! HTTP `Range:` request-header parser.
//!
//! Pure helper: takes an inbound `Range:` header value plus the total
//! resource size in bytes, returns a [`RangeOutcome`] describing how the
//! caller should respond.
//!
//! Shared by the meta-share hull and every transport plugin so the
//! `<video>` element can seek and resume downloads identically whichever
//! process serves the bytes. Moved verbatim from meta-share
//! `crates/meta-share/src/api/range.rs`.
//!
//! ## Supported syntax
//!
//! RFC 7233 §3.1 `bytes=`-prefixed single-range only. Multi-range
//! requests (`bytes=0-99,200-299`) are surfaced as
//! [`RangeOutcome::Unsupported`] so the caller can fall back to `200 OK`
//! with the full body — the RFC permits ignoring multi-range, and
//! every common client (browser video element, curl, wget) accepts the
//! degraded response.
//!
//! ### Returned shape
//!
//! - `bytes=N-M` → `Ok((N, min(M, total-1)))`
//! - `bytes=N-` → `Ok((N, total-1))`
//! - `bytes=-M` → `Ok((max(0, total-M), total-1))` (suffix)
//! - `bytes=N-` with `N >= total` → `Unsatisfiable`
//! - `bytes=N-M` with `N > M` → `Unsatisfiable`
//! - missing / blank / malformed → `Unsatisfiable` for non-`Absent`,
//! `Absent` only for the literal "no header" case (callers pass
//! `Option<&HeaderValue>`).
//!
//! The function is total: it never panics, never allocates, and always
//! returns one of the four variants.
use axum::http::HeaderValue;
/// Outcome of parsing an inbound `Range:` header.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RangeOutcome {
/// Header absent on the request — caller serves `200 OK` with the
/// full body.
Absent,
/// Single-range request parsed cleanly. Inclusive `[start, end]`
/// clamped to `[0, total-1]`.
Ok { start: u64, end: u64 },
/// Multi-range or unsupported syntax — caller serves `200 OK` with
/// the full body (the RFC permits ignoring; browser video and curl
/// both fall through to a full GET on a follow-up request).
Unsupported,
/// Malformed range, reversed range, or `start >= total` — caller
/// serves `416 Range Not Satisfiable` with
/// `Content-Range: bytes */{total}`.
Unsatisfiable,
}
/// Parse a `Range:` header into a [`RangeOutcome`]. `header` is the raw
/// header value (or `None` if the request didn't carry one). `total` is
/// the resource's full size in bytes — used to clamp open / suffix
/// ranges and to detect out-of-bounds requests.
pub fn parse_range_header(header: Option<&HeaderValue>, total: u64) -> RangeOutcome {
let Some(value) = header else {
return RangeOutcome::Absent;
};
let Ok(s) = value.to_str() else {
return RangeOutcome::Unsatisfiable;
};
let s = s.trim();
// Strip the `bytes=` unit prefix. Anything else is unsupported
// (per RFC the unit is extensible, but in practice `bytes` is the
// only one any browser sends).
let Some(rest) = s.strip_prefix("bytes=") else {
return RangeOutcome::Unsatisfiable;
};
let rest = rest.trim();
if rest.is_empty() {
return RangeOutcome::Unsatisfiable;
}
// Multi-range: caller serves 200 with the full body.
if rest.contains(',') {
return RangeOutcome::Unsupported;
}
// Must contain exactly one '-' separator.
let (lhs, rhs) = match rest.split_once('-') {
Some(parts) => parts,
None => return RangeOutcome::Unsatisfiable,
};
let lhs = lhs.trim();
let rhs = rhs.trim();
// Three valid shapes:
// - "N-M" bounded range
// - "N-" open-ended (start known, end = total-1)
// - "-M" suffix (last M bytes)
if lhs.is_empty() && rhs.is_empty() {
return RangeOutcome::Unsatisfiable;
}
// Edge case: file is empty. Any range against a 0-byte resource is
// unsatisfiable (RFC 7233 §4.4).
if total == 0 {
return RangeOutcome::Unsatisfiable;
}
if lhs.is_empty() {
// Suffix range. `rhs` is the suffix length.
let Ok(suffix_len) = rhs.parse::<u64>() else {
return RangeOutcome::Unsatisfiable;
};
if suffix_len == 0 {
return RangeOutcome::Unsatisfiable;
}
let start = total.saturating_sub(suffix_len);
return RangeOutcome::Ok {
start,
end: total - 1,
};
}
let Ok(start) = lhs.parse::<u64>() else {
return RangeOutcome::Unsatisfiable;
};
if start >= total {
return RangeOutcome::Unsatisfiable;
}
let end = if rhs.is_empty() {
total - 1
} else {
let Ok(parsed_end) = rhs.parse::<u64>() else {
return RangeOutcome::Unsatisfiable;
};
if parsed_end < start {
return RangeOutcome::Unsatisfiable;
}
// Clamp end to total-1 (RFC: "the last byte position is at most
// total-1").
parsed_end.min(total - 1)
};
RangeOutcome::Ok { start, end }
}
#[cfg(test)]
mod tests {
use super::*;
fn hv(s: &str) -> HeaderValue {
HeaderValue::from_str(s).expect("static header value")
}
#[test]
fn absent_when_no_header() {
assert_eq!(parse_range_header(None, 1000), RangeOutcome::Absent);
}
#[test]
fn closed_range_in_bounds() {
assert_eq!(
parse_range_header(Some(&hv("bytes=0-99")), 1000),
RangeOutcome::Ok { start: 0, end: 99 }
);
}
#[test]
fn open_end_clamped_to_total_minus_one() {
assert_eq!(
parse_range_header(Some(&hv("bytes=100-")), 1000),
RangeOutcome::Ok { start: 100, end: 999 }
);
}
#[test]
fn suffix_range() {
assert_eq!(
parse_range_header(Some(&hv("bytes=-512")), 1000),
RangeOutcome::Ok { start: 488, end: 999 }
);
}
#[test]
fn single_byte_range() {
assert_eq!(
parse_range_header(Some(&hv("bytes=0-0")), 1000),
RangeOutcome::Ok { start: 0, end: 0 }
);
assert_eq!(
parse_range_header(Some(&hv("bytes=999-999")), 1000),
RangeOutcome::Ok { start: 999, end: 999 }
);
}
#[test]
fn suffix_clamps_to_whole_file_when_larger_than_total() {
assert_eq!(
parse_range_header(Some(&hv("bytes=-99999")), 1000),
RangeOutcome::Ok { start: 0, end: 999 }
);
}
#[test]
fn end_clamped_when_above_total() {
assert_eq!(
parse_range_header(Some(&hv("bytes=0-1000")), 1000),
RangeOutcome::Ok { start: 0, end: 999 }
);
assert_eq!(
parse_range_header(Some(&hv("bytes=0-999999")), 1000),
RangeOutcome::Ok { start: 0, end: 999 }
);
}
#[test]
fn start_at_or_past_total_is_unsatisfiable() {
assert_eq!(
parse_range_header(Some(&hv("bytes=1000-")), 1000),
RangeOutcome::Unsatisfiable
);
assert_eq!(
parse_range_header(Some(&hv("bytes=2000-3000")), 1000),
RangeOutcome::Unsatisfiable
);
}
#[test]
fn reversed_range_is_unsatisfiable() {
assert_eq!(
parse_range_header(Some(&hv("bytes=10-9")), 1000),
RangeOutcome::Unsatisfiable
);
}
#[test]
fn garbage_input_is_unsatisfiable() {
assert_eq!(
parse_range_header(Some(&hv("bytes=abc")), 1000),
RangeOutcome::Unsatisfiable
);
assert_eq!(
parse_range_header(Some(&hv("bytes=")), 1000),
RangeOutcome::Unsatisfiable
);
assert_eq!(
parse_range_header(Some(&hv("blocks=0-99")), 1000),
RangeOutcome::Unsatisfiable
);
assert_eq!(
parse_range_header(Some(&hv("0-99")), 1000),
RangeOutcome::Unsatisfiable
);
}
#[test]
fn multi_range_is_unsupported() {
assert_eq!(
parse_range_header(Some(&hv("bytes=0-50,100-150")), 1000),
RangeOutcome::Unsupported
);
assert_eq!(
parse_range_header(Some(&hv("bytes=0-50, 100-150")), 1000),
RangeOutcome::Unsupported
);
}
#[test]
fn empty_resource_is_always_unsatisfiable() {
assert_eq!(
parse_range_header(Some(&hv("bytes=0-99")), 0),
RangeOutcome::Unsatisfiable
);
assert_eq!(
parse_range_header(Some(&hv("bytes=-10")), 0),
RangeOutcome::Unsatisfiable
);
}
#[test]
fn zero_length_suffix_is_unsatisfiable() {
assert_eq!(
parse_range_header(Some(&hv("bytes=-0")), 1000),
RangeOutcome::Unsatisfiable
);
}
}
+170
View File
@@ -0,0 +1,170 @@
//! `serve_transport` — the axum harness every transport plugin binary runs.
//!
//! # Common routes
//!
//! - `GET /manifest` → [`Manifest`](super::dto::Manifest)
//! - `GET /health` → [`Health`](super::dto::Health)
//! - `GET /raw/:cid` → the tier's byte response, verbatim
//! - `GET /jobs` → [`Jobs`](super::dto::Jobs)
//! - `DELETE /jobs/:cid?keep=` → [`Deleted`](super::dto::Deleted)
//! - `POST /reconcile` → [`ReconcileReport`](super::dto::ReconcileReport)
//! - `PUT /focus` ← [`FocusSnapshot`](super::focus::FocusSnapshot), `204`
//! - `GET /focus` → the snapshot the plugin currently obeys
//!
//! plus the `/config*` routes when [`TransportPlugin::config`] is `Some` (see
//! [`ConfigPlane`](super::config::ConfigPlane)), plus whatever
//! [`TransportPlugin::extra_routes`] adds.
use std::net::SocketAddr;
use std::sync::Arc;
use axum::extract::{Path, Query, State};
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use tracing::info;
use super::dto::{DeleteQuery, Jobs, ReconcileRequest};
use super::focus::FocusSnapshot;
use super::plugin::TransportPlugin;
type Shared = Arc<dyn TransportPlugin>;
/// The common router for `plugin`, merged with its extra routes.
pub fn router<P: TransportPlugin>(plugin: Arc<P>) -> Router {
let extra = Arc::clone(&plugin).extra_routes();
let config = plugin.config().map(|c| c.routes());
let shared: Shared = plugin;
let r = Router::new()
.route("/manifest", get(manifest))
.route("/health", get(health))
.route("/raw/:cid", get(raw))
.route("/jobs", get(jobs))
.route("/jobs/:cid", delete(delete_job))
.route("/reconcile", post(reconcile))
.route("/focus", get(get_focus).put(put_focus))
.with_state(shared)
.merge(extra);
match config {
Some(c) => r.merge(c),
None => r,
}
}
/// Bind `listen` and serve `plugin` until SIGTERM / Ctrl-C.
pub async fn serve_transport<P: TransportPlugin>(
plugin: Arc<P>,
listen: SocketAddr,
) -> anyhow::Result<()> {
let manifest = plugin.manifest();
let id = manifest.id.clone();
let advertised = Arc::clone(&plugin);
let app = router(plugin);
let listener = tokio::net::TcpListener::bind(listen).await?;
info!(%listen, plugin = %id, "transport plugin listening");
// Beacon v2: advertise once bound; say `bye` on clean shutdown so the hull
// drops us at once instead of after the liveness window.
let port = listen.port();
// `node.name` is a short machine name; the human description stays in the
// manifest (`implementation`), which a consumer fetches over HTTP.
let beacon_name = std::env::var("SERVICE_NAME")
.ok()
.filter(|s| !s.is_empty())
.unwrap_or_else(|| format!("meta-transport-{}", manifest.id));
let beacon = crate::beacon::advertise_plugin(&beacon_name, &manifest.version, move || {
beacon_resource(advertised.as_ref(), port)
});
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await?;
if let Some(b) = beacon {
b.bye().await;
}
Ok(())
}
/// The transport's beacon v2 resource: id = manifest id, cap
/// `metamesh.transport/<id>@<contract>`, `rev` = hash of the served manifest.
fn beacon_resource<P: TransportPlugin + ?Sized>(plugin: &P, listen_port: u16) -> crate::beacon::Resource {
use crate::beacon::{advertise_url, binds_from_env, caps, rev_of, Resource};
let mut m = plugin.manifest();
m.config = plugin.config().is_some();
let mut r = Resource::new(m.id.clone(), [caps::transport(&m.id, m.contract)])
.with_endpoint("http", advertise_url(listen_port))
.with_endpoint("manifest", "/manifest");
r.rev = Some(rev_of(&m));
r.binds = binds_from_env();
r
}
/// Resolves on SIGTERM (docker stop) or Ctrl-C.
pub async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let term = async {
if let Ok(mut s) =
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
{
s.recv().await;
}
};
#[cfg(not(unix))]
let term = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {},
_ = term => {},
}
}
async fn manifest(State(p): State<Shared>) -> Response {
let mut m = p.manifest();
m.config = p.config().is_some();
Json(m).into_response()
}
async fn health(State(p): State<Shared>) -> Response {
let h = p.health().await;
let code = if h.ok { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE };
(code, Json(h)).into_response()
}
async fn raw(State(p): State<Shared>, Path(cid): Path<String>, headers: HeaderMap) -> Response {
p.raw(cid, headers).await
}
async fn jobs(State(p): State<Shared>) -> Response {
match p.jobs().await {
Ok(jobs) => Json(Jobs { jobs }).into_response(),
Err(e) => e.into_response(),
}
}
async fn delete_job(
State(p): State<Shared>,
Path(cid): Path<String>,
Query(q): Query<DeleteQuery>,
) -> Response {
match p.delete(cid, q.keep_list()).await {
Ok(d) => Json(d).into_response(),
Err(e) => e.into_response(),
}
}
async fn reconcile(State(p): State<Shared>, Json(req): Json<ReconcileRequest>) -> Response {
match p.reconcile(req).await {
Ok(r) => Json(r).into_response(),
Err(e) => e.into_response(),
}
}
async fn get_focus(State(p): State<Shared>) -> Response {
Json(p.focus().snapshot()).into_response()
}
async fn put_focus(State(p): State<Shared>, Json(snap): Json<FocusSnapshot>) -> StatusCode {
p.focus().apply(snap);
StatusCode::NO_CONTENT
}
+218
View File
@@ -0,0 +1,218 @@
//! Conformance checks any transport plugin must pass — ours or a competing
//! implementation (a qBittorrent-backed torrent plugin, say). Run them against a
//! live plugin URL, or in a unit test via [`spawn_local`].
//!
//! The checks cover the *common* contract only. A plugin's byte behaviour is
//! verified by its own tests and by meta-share's contract suite against the
//! hull, which is where "identical to the monolith" is decided.
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::Duration;
use super::client::RemoteTransport;
use super::dto::{Deleted, CONTRACT_VERSION};
use super::focus::{FocusSnapshot, FocusTitle};
use super::plugin::TransportPlugin;
/// A well-formed cid no plugin holds a job for: the raw/sha2-256 CID of the
/// empty block.
pub const UNKNOWN_CID: &str = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku";
/// Serve `plugin` on an ephemeral localhost port. Returns its address; the
/// server lives until the runtime shuts down.
pub async fn spawn_local<P: TransportPlugin>(plugin: Arc<P>) -> SocketAddr {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind");
let addr = listener.local_addr().expect("addr");
let app = super::serve::router(plugin);
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
addr
}
/// Run every check against the plugin at `base`. `Ok(())` or the list of
/// failures, each a one-line explanation.
pub async fn conformance(base: &str) -> Result<(), Vec<String>> {
let http = reqwest::Client::new();
let t = RemoteTransport::new("under-test", base, http.clone())
.with_control_timeout(Duration::from_secs(10));
let mut fails = Vec::new();
let mut has_config = false;
match t.manifest().await {
Ok(m) => {
has_config = m.config;
if m.id.is_empty() {
fails.push("manifest.id is empty".into());
}
if m.contract != CONTRACT_VERSION {
fails.push(format!("manifest.contract {} != {CONTRACT_VERSION}", m.contract));
}
if !m.capabilities.fetch {
fails.push("manifest.capabilities.fetch is false — a transport must fetch".into());
}
}
Err(e) => fails.push(format!("GET /manifest: {e}")),
}
match http.get(t.url("/health")).send().await {
Ok(r) if r.status().is_success() || r.status().as_u16() == 503 => {
if r.json::<super::dto::Health>().await.is_err() {
fails.push("GET /health body is not a Health".into());
}
}
Ok(r) => fails.push(format!("GET /health: status {}", r.status())),
Err(e) => fails.push(format!("GET /health: {e}")),
}
if let Err(e) = t.jobs().await {
fails.push(format!("GET /jobs: {e}"));
}
let snap = FocusSnapshot {
enabled: true,
bg_rate_bytes: 4321,
titles: vec![FocusTitle {
group: "conformance-group".into(),
cids: vec!["conformance-cid".into()],
infohashes: vec![],
}],
};
match t.push_focus(&snap).await {
Ok(()) => match t.get_json::<FocusSnapshot>("/focus").await {
Ok(back) if back == snap => {}
Ok(back) => fails.push(format!("GET /focus returned {back:?}, pushed {snap:?}")),
Err(e) => fails.push(format!("GET /focus: {e}")),
},
Err(e) => fails.push(format!("PUT /focus: {e}")),
}
// Leave the plugin unfocused.
let _ = t.push_focus(&FocusSnapshot::default()).await;
match http.delete(t.url(&format!("/jobs/{UNKNOWN_CID}"))).send().await {
Ok(r) if r.status().as_u16() == 404 => {}
Ok(r) if r.status().is_success() => match r.json::<Deleted>().await {
Ok(d) if d.bytes_freed == 0 => {}
Ok(d) => fails.push(format!("DELETE unknown job freed {} bytes", d.bytes_freed)),
Err(e) => fails.push(format!("DELETE unknown job: body: {e}")),
},
Ok(r) => fails.push(format!("DELETE unknown job: status {}", r.status())),
Err(e) => fails.push(format!("DELETE unknown job: {e}")),
}
match http.get(t.url("/raw/not-a-cid")).send().await {
Ok(r) if r.status().is_client_error() => {
let v: Option<serde_json::Value> = r.json().await.ok();
if !v.as_ref().is_some_and(|v| v.get("error").is_some()) {
fails.push("GET /raw/not-a-cid: 4xx without an {\"error\"} body".into());
}
}
Ok(r) => fails.push(format!("GET /raw/not-a-cid: expected 4xx, got {}", r.status())),
Err(e) => fails.push(format!("GET /raw/not-a-cid: {e}")),
}
if has_config {
match http.get(t.url("/config/schema")).send().await {
Ok(r) if r.status().is_success() => {
if r.json::<crate::config::ConfigSchema>().await.is_err() {
fails.push("GET /config/schema body is not a ConfigSchema".into());
}
}
Ok(r) => fails.push(format!("GET /config/schema: status {}", r.status())),
Err(e) => fails.push(format!("GET /config/schema: {e}")),
}
match http.get(t.url("/config/values")).send().await {
Ok(r) if r.status().is_success() => match r.json::<serde_json::Value>().await {
Ok(v) if v.is_object() => {}
_ => fails.push("GET /config/values is not a JSON object".into()),
},
Ok(r) => fails.push(format!("GET /config/values: status {}", r.status())),
Err(e) => fails.push(format!("GET /config/values: {e}")),
}
match http.get(t.url("/config")).send().await {
Ok(r) if r.status().is_success() => {}
Ok(r) => fails.push(format!("GET /config: status {}", r.status())),
Err(e) => fails.push(format!("GET /config: {e}")),
}
}
if fails.is_empty() {
Ok(())
} else {
Err(fails)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::transport::dto::{Capabilities, Job, Manifest, ReconcileReport, ReconcileRequest};
use crate::transport::error::ApiError;
use crate::transport::focus::FocusView;
use async_trait::async_trait;
use axum::http::HeaderMap;
use axum::response::{IntoResponse, Response};
/// The smallest plugin that should pass: holds nothing, fetches nothing.
struct Empty {
focus: Arc<FocusView>,
config: Option<Arc<crate::transport::config::ConfigPlane>>,
}
#[async_trait]
impl TransportPlugin for Empty {
fn manifest(&self) -> Manifest {
Manifest {
id: "empty".into(),
implementation: "testkit-stub".into(),
version: "0".into(),
contract: CONTRACT_VERSION,
capabilities: Capabilities { fetch: true, share: false },
config: false,
}
}
async fn raw(&self, _cid: String, _h: HeaderMap) -> Response {
ApiError::BadRequest("unsupported cid".into()).into_response()
}
async fn jobs(&self) -> Result<Vec<Job>, ApiError> {
Ok(vec![])
}
async fn delete(&self, _cid: String, _keep: Vec<String>) -> Result<Deleted, ApiError> {
Ok(Deleted::default())
}
async fn reconcile(&self, _r: ReconcileRequest) -> Result<ReconcileReport, ApiError> {
Ok(ReconcileReport::default())
}
fn focus(&self) -> &Arc<FocusView> {
&self.focus
}
fn config(&self) -> Option<Arc<crate::transport::config::ConfigPlane>> {
self.config.clone()
}
}
#[tokio::test]
async fn the_empty_plugin_conforms() {
let addr = spawn_local(Arc::new(Empty { focus: FocusView::new(), config: None })).await;
conformance(&format!("http://{addr}")).await.unwrap();
}
#[tokio::test]
async fn a_plugin_with_a_config_plane_conforms_and_says_so() {
use crate::config::{ConfigField, ConfigSchema};
use crate::transport::config::ConfigPlane;
let dir = tempfile::tempdir().unwrap();
let plane = ConfigPlane::new(
ConfigSchema { fields: vec![ConfigField::secret("key", "Key")] },
dir.path(),
serde_json::json!({}),
)
.without_restart();
let addr = spawn_local(Arc::new(Empty { focus: FocusView::new(), config: Some(Arc::new(plane)) })).await;
let base = format!("http://{addr}");
conformance(&base).await.unwrap();
let m: Manifest = reqwest::get(format!("{base}/manifest")).await.unwrap().json().await.unwrap();
assert!(m.config, "the harness must advertise the config plane");
}
}
+263
View File
@@ -0,0 +1,263 @@
//! BitTorrent-specific half of the contract: the torrent plugin's extra routes,
//! and the magnet helpers both sides use.
//!
//! Extra routes (all under `/torrent/`):
//!
//! - `GET /torrent/inventory` → [`Inventory`] — every managed torrent,
//! with have-bytes, per-file progress and live stats. One call per eviction
//! pass / dashboard poll instead of one per row.
//! - `GET /torrent/status/:cid` → `Option<`[`LiveStats`]`>` — peeks only,
//! never starts the session (meta-watch polls it while connecting).
//! - `GET /torrent/peers/:cid?budget_ms=&cap=` → [`PeerCount`] — query-only
//! mainline-DHT population for the availability probe.
//! - `POST /torrent/confirm` ← [`ConfirmRequest`] → [`ConfirmResponse`]
//! — metadata-only reachability probe (no torrent kept, no seed row).
//! - `PUT /torrent/commits` ← [`Commits`] — the watch-commit intents.
//! The hull owns the intent (`seeds.json`); the plugin's supervisor owns the
//! fill (disk floor, concurrency cap, stall hysteresis).
use serde::{Deserialize, Serialize};
/// Live transfer stats for one torrent. Serialises exactly like meta-share's
/// former `bt::SeedLiveStats`, which `/api/file/:cid/status` and `/api/seeds`
/// still emit.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct LiveStats {
/// librqbit torrent state: `live` | `paused` | `initializing` | `error`.
pub state: String,
pub down_bps: u64,
pub up_bps: u64,
pub uploaded_bytes: u64,
pub downloaded_bytes: u64,
pub total_bytes: u64,
pub ratio: f64,
pub peers_live: usize,
pub peers_seen: usize,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct FileProgress {
pub index: usize,
/// File length from the torrent metadata.
pub len: u64,
/// Bytes of it on disk.
pub have: u64,
}
impl FileProgress {
pub fn complete(&self) -> bool {
self.have >= self.len
}
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct TorrentInfo {
/// Lowercase hex v1 infohash.
pub infohash: String,
#[serde(default)]
pub name: Option<String>,
/// librqbit have-bytes — the eviction size of the unit.
pub have_bytes: u64,
pub streaming_only: bool,
/// Metadata resolved (`files` is empty until it is).
pub metadata: bool,
#[serde(default)]
pub files: Vec<FileProgress>,
pub stats: LiveStats,
}
impl TorrentInfo {
/// Is file `index` fully downloaded? `None` when unknowable yet.
pub fn file_complete(&self, index: usize) -> Option<bool> {
self.files.iter().find(|f| f.index == index).map(FileProgress::complete)
}
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct Inventory {
/// Is the librqbit session up? When it isn't, `torrents` is empty and the
/// hull falls back to row lengths (the session is lazy on purpose).
pub session_up: bool,
/// Does the session directory hold persisted torrents?
pub persisted: bool,
pub torrents: Vec<TorrentInfo>,
}
impl Inventory {
pub fn get(&self, infohash_hex: &str) -> Option<&TorrentInfo> {
self.torrents.iter().find(|t| t.infohash == infohash_hex)
}
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct PeerCount {
/// Distinct DHT peers seen; `None` when the DHT couldn't be queried.
pub peers: Option<usize>,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct ConfirmRequest {
pub magnet: String,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, Default)]
pub struct ConfirmResponse {
/// Length of file 0.
pub len: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct CommitIntent {
pub cid: String,
/// The seed row's `added_at` — the supervisor's recency order.
pub added_at: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq, Eq)]
pub struct Commits {
pub intents: Vec<CommitIntent>,
}
// ---------------------------------------------------------------------------
// Magnet helpers (moved verbatim from meta-share `bt.rs`)
// ---------------------------------------------------------------------------
/// A bare-infohash magnet: DHT resolves peers from the infohash alone.
pub fn build_magnet(infohash20: &[u8; 20]) -> String {
format!("magnet:?xt=urn:btih:{}", super::cid::to_hex(infohash20))
}
/// High-uptime open trackers appended to every magnet before librqbit sees it,
/// so a trackerless indexer magnet doesn't leave peer discovery to DHT alone.
pub const DEFAULT_TRACKERS: &[&str] = &[
"udp://tracker.opentrackr.org:1337/announce",
"udp://open.stealth.si:80/announce",
"udp://tracker.openbittorrent.com:6969/announce",
"udp://exodus.desync.com:6969/announce",
"udp://tracker.torrent.eu.org:451/announce",
"udp://open.demonii.com:1337/announce",
"http://nyaa.tracker.wf:7777/announce",
];
fn encode_tr(url: &str) -> String {
let mut out = String::with_capacity(url.len() * 3);
for b in url.bytes() {
match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
out.push(b as char)
}
_ => out.push_str(&format!("%{b:02X}")),
}
}
out
}
/// Append [`DEFAULT_TRACKERS`] to `magnet`, skipping any already present. A
/// non-`magnet:` string is returned unchanged.
pub fn with_default_trackers(mut magnet: String) -> String {
if !magnet.starts_with("magnet:") {
return magnet;
}
for tr in DEFAULT_TRACKERS {
let enc = encode_tr(tr);
if magnet.contains(&enc) || magnet.contains(tr) {
continue;
}
magnet.push_str("&tr=");
magnet.push_str(&enc);
}
magnet
}
fn decode_tr(s: &str) -> String {
let bytes = s.as_bytes();
let mut out: Vec<u8> = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%' && i + 2 < bytes.len() {
let hi = (bytes[i + 1] as char).to_digit(16);
let lo = (bytes[i + 2] as char).to_digit(16);
if let (Some(hi), Some(lo)) = (hi, lo) {
out.push((hi * 16 + lo) as u8);
i += 3;
continue;
}
}
out.push(bytes[i]);
i += 1;
}
String::from_utf8_lossy(&out).into_owned()
}
/// The decoded, de-duplicated `tr=` announce URLs of a magnet.
pub fn parse_trackers(magnet: &str) -> Vec<String> {
if !magnet.starts_with("magnet:") {
return Vec::new();
}
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
let query = magnet.splitn(2, '?').nth(1).unwrap_or("");
for pair in query.split('&') {
if let Some(val) = pair.strip_prefix("tr=") {
let url = decode_tr(val);
if !url.is_empty() && seen.insert(url.clone()) {
out.push(url);
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn build_magnet_includes_infohash() {
assert_eq!(
build_magnet(&[0xab; 20]),
"magnet:?xt=urn:btih:abababababababababababababababababababab"
);
}
#[test]
fn with_default_trackers_appends_to_trackerless_magnet() {
let bare = "magnet:?xt=urn:btih:abababababababababababababababababababab&dn=Foo".to_string();
let out = with_default_trackers(bare);
assert!(out.contains("&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce"));
assert!(out.contains("&tr=http%3A%2F%2Fnyaa.tracker.wf%3A7777%2Fannounce"));
assert!(out.contains("&dn=Foo"));
}
#[test]
fn with_default_trackers_dedups_existing() {
let pre = "magnet:?xt=urn:btih:abababababababababababababababababababab\
&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337%2Fannounce"
.to_string();
let out = with_default_trackers(pre);
assert_eq!(out.matches("tracker.opentrackr.org%3A1337").count(), 1);
}
#[test]
fn with_default_trackers_leaves_non_magnet_untouched() {
assert_eq!(with_default_trackers("not-a-magnet".to_string()), "not-a-magnet");
}
#[test]
fn parse_trackers_round_trips_the_defaults() {
let m = with_default_trackers(build_magnet(&[1; 20]));
assert_eq!(parse_trackers(&m), DEFAULT_TRACKERS.iter().map(|s| s.to_string()).collect::<Vec<_>>());
assert!(parse_trackers("not-a-magnet").is_empty());
}
#[test]
fn live_stats_serialise_with_the_legacy_field_names() {
let v = serde_json::to_value(LiveStats::default()).unwrap();
for k in [
"state", "down_bps", "up_bps", "uploaded_bytes", "downloaded_bytes",
"total_bytes", "ratio", "peers_live", "peers_seen",
] {
assert!(v.get(k).is_some(), "missing {k}");
}
}
}