improvement

This commit is contained in:
Worph
2026-09-13 14:30:54 +02:00
parent 60c4e855a5
commit 1c9d415a1e
6 changed files with 496 additions and 11 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "meta-feeder-sdk" name = "meta-feeder-sdk"
version = "0.1.0" version = "1.3.0"
edition = "2021" edition = "2021"
rust-version = "1.85" rust-version = "1.85"
license = "MIT" license = "MIT"
+40 -7
View File
@@ -17,7 +17,7 @@ bytes; it never joins the swarm.
```toml ```toml
[dependencies] [dependencies]
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v0.1.0" } meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.3.0" }
``` ```
Not on crates.io yet — the contract is still moving, so consumers pin a tag. Not on crates.io yet — the contract is still moving, so consumers pin a tag.
@@ -47,8 +47,32 @@ Four required methods; everything else has a default:
| `compute_outcomes` | resolve a `record_id` to `HashOutcome`s | `bytes: None` for metadata-only sources | | `compute_outcomes` | resolve a `record_id` to `HashOutcome`s | `bytes: None` for metadata-only sources |
Defaults you can override: `handle_query_stream`, `handle_fetch`, `health`, Defaults you can override: `handle_query_stream`, `handle_fetch`, `health`,
`served_file_types`, `served_content_kinds`, `config_schema`, `config_values`, `served_file_types`, `served_content_kinds`, `package`, `redeems`,
`get_blob`. `config_schema`, `config_values`, `get_blob`.
### Redeeming locators (1.3.0)
A plugin can **redeem** a locator family — turn a locator cid into the bytes it
names, usually spending a metered quota (a Newznab `.nzb` grab, an OpenSubtitles
download):
1. Return a `RedeemClaim` from `redeems()` — `RedeemClaim::nzb_release(hosts)` or
`RedeemClaim::provider_file(sources)` — reflecting the *current* config (no
credential → no claim). Served per plugin in `GET /manifest` and, for polling,
at `GET /redeems` → `{"redeems": {"<upstream_id>": [claim…]}}`.
2. Name a stable `package()` — the `/files/plugin/<package>/` folder the gateway
stores the bytes under.
3. In `compute_outcomes`, accept the locator cid as `record_id` and return ONE
`Sha2_256` outcome: `hash = hash::compute_ipfs_cid(&bytes)`, `bytes`,
`file_extension`, and optionally a `record` whose fields the gateway merges
onto the locator record. Return `NotFound` when the cid is not yours (e.g. no
key for that host) so the gateway can try the next claimer; `RateLimited` when
the quota is spent.
The gateway calls this only on a real play, one claimer at a time, and writes
the bytes' cid onto the locator record under the claim's `field` (`manifest` /
`file`), so each file's quota is spent once. Decode helpers:
`hash::decode_nzb_release_cid`, `hash::decode_provider_file_cid`, `hash::codec_of`.
### The routing gate ### The routing gate
@@ -101,6 +125,9 @@ into the bitswap blockstore.
The last two carry an identity multihash. Anything that dispatches on the The last two carry an identity multihash. Anything that dispatches on the
multihash instead of the **codec** will silently mis-rank them. multihash instead of the **codec** will silently mis-rank them.
`hash` also encodes/decodes the redeemable locators: `nzb-release` (`0x1005`)
and `provider-file` (`0x100A`, `(source, id)` — `docs/cid-formats.md` §8).
## Layout ## Layout
| module | what | | module | what |
@@ -121,13 +148,19 @@ multihash instead of the **codec** will silently mis-rank them.
cargo build && cargo test cargo build && cargo test
``` ```
To work on the SDK from inside a feeder checkout without pushing a tag, add a To test a feeder against an SDK change **before its tag is pushed**: cargo will
gitignored `.cargo/config.toml` in the feeder: not resolve an unpushed tag, not even under a `[patch]` override ("failed to
find tag"). Build a scratch copy of the feeder (outside its git tree) with the
dependency swapped for a path, and run `cargo test` there:
```toml ```bash
paths = ["../meta-feeder-sdk"] mkdir -p /tmp/feeder && (cd meta-feeder-x && tar cf - --exclude=./target --exclude=./.git .) | (cd /tmp/feeder && tar xf -)
sed -i 's#meta-feeder-sdk = { git = .*#meta-feeder-sdk = { path = "/abs/path/to/meta-feeder-sdk" }#' /tmp/feeder/feeder-plugin/*/Cargo.toml
(cd /tmp/feeder && cargo test)
``` ```
Never commit the path form: each feeder repo is its own Docker build context.
## Context ## Context
Design rationale, the feeder roster, and the naming rules live in the meta-root: Design rationale, the feeder roster, and the naming rules live in the meta-root:
+183 -1
View File
@@ -206,6 +206,76 @@ pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option<Strin
Some(format!("b{}", base32_lower_no_padding(&wire))) Some(format!("b{}", base32_lower_no_padding(&wire)))
} }
/// Decode a self-describing `nzb-release` cid into `(api_base, id)` — the exact
/// inverse of [`compute_nzb_release_cid`], string-level (no `cid` crate).
///
/// `api_base` is scheme-less authority + optional path (`api.nzbgeek.info/api`);
/// match credentials on its authority (split at the first `/`), grab against
/// `https://{api_base}`. `id` is lowercase hex, re-encoded from the digest — so
/// an **odd-length** id comes back left-padded with `0` (the encoder pads it),
/// which Newznab `t=get` accepts as the same release.
///
/// `None` on a wrong codec, a non-identity multihash, or a malformed digest.
pub fn decode_nzb_release_cid(cid: &str) -> Option<(String, String)> {
let digest = identity_locator_digest(cid, NZB_RELEASE_CODEC)?;
let mut pos = 0usize;
let base_len = read_varint(&digest, &mut pos)? as usize;
let rest = digest.get(pos..)?;
if rest.len() <= base_len {
return None; // need the base AND at least one id byte
}
let (base, id) = rest.split_at(base_len);
let api_base = std::str::from_utf8(base).ok()?.to_string();
let id_hex: String = id.iter().map(|b| format!("{b:02x}")).collect();
Some((api_base, id_hex))
}
/// The multicodec of a base32 (`b…`) CIDv1, or `None` for anything else
/// (CIDv0 `Qm…`, other multibases, garbage). Dispatch on this — never on the
/// multihash, which is identity (`0x00`) for every locator family.
pub fn codec_of(cid: &str) -> Option<u64> {
let wire = base32_lower_decode(cid.strip_prefix('b')?)?;
let mut pos = 0usize;
if read_varint(&wire, &mut pos)? != 1 {
return None;
}
read_varint(&wire, &mut pos)
}
/// The digest of a well-formed identity-multihash CIDv1 carrying `codec`, or
/// `None`. Shared by the locator decoders.
fn identity_locator_digest(cid: &str, codec: u64) -> Option<Vec<u8>> {
let wire = base32_lower_decode(cid.strip_prefix('b')?)?;
let mut pos = 0usize;
if read_varint(&wire, &mut pos)? != 1
|| read_varint(&wire, &mut pos)? != codec
|| read_varint(&wire, &mut pos)? != 0x00
{
return None;
}
let len = read_varint(&wire, &mut pos)? as usize;
let digest = wire.get(pos..)?;
(digest.len() == len).then(|| digest.to_vec())
}
/// Unsigned LEB128 varint at `*pos`, advancing it.
fn read_varint(bytes: &[u8], pos: &mut usize) -> Option<u64> {
let mut value = 0u64;
let mut shift = 0u32;
loop {
let byte = *bytes.get(*pos)?;
*pos += 1;
value |= u64::from(byte & 0x7f) << shift;
if byte & 0x80 == 0 {
return Some(value);
}
shift += 7;
if shift >= 64 {
return None;
}
}
}
/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the /// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the
/// id is all-hex by construction; odd-length ids are left-padded with a `0` /// id is all-hex by construction; odd-length ids are left-padded with a `0`
/// nibble. Returns `None` on any non-hex byte (defensive). /// nibble. Returns `None` on any non-hex byte (defensive).
@@ -399,8 +469,39 @@ pub fn compute_ext_play_cid(url: &str) -> Option<String> {
Some(identity_locator_cid(EXT_PLAY_CODEC, digest)) Some(identity_locator_cid(EXT_PLAY_CODEC, digest))
} }
/// Custom multicodec for a fetchable **`provider-file`** locator —
/// `docs/cid-formats.md` §8. One codec for every provider whose bytes are
/// fetched the same way; the provider lives inside the digest:
/// `varint(len(source)) ‖ source ‖ id`. Framing byte-identical to `card`
/// (`0x1007`) — only the codec differs — but the byte path *redeems* it (through
/// the claiming feeder's `/compute`) instead of 404ing.
pub const PROVIDER_FILE_CODEC: u64 = 0x100A;
/// Encode a `provider-file` CID from a registered source token
/// (`opensubtitles`) and that source's id (`file:7061834`). `None` on an empty
/// part or a digest past 64 bytes — the caller drops that row rather than emit
/// an address no peer can parse.
pub fn compute_provider_file_cid(source: &str, id: &str) -> Option<String> {
kinded_locator_cid(PROVIDER_FILE_CODEC, source, id)
}
/// Inverse of [`compute_provider_file_cid`]: `(source, id)`. Matches the
/// **codec slot** — a `card` with identical framing does not decode here.
pub fn decode_provider_file_cid(cid: &str) -> Option<(String, String)> {
let digest = identity_locator_digest(cid, PROVIDER_FILE_CODEC)?;
let mut pos = 0usize;
let source_len = read_varint(&digest, &mut pos)? as usize;
let source = digest.get(pos..pos.checked_add(source_len)?)?;
let id = digest.get(pos + source_len..)?;
Some((
String::from_utf8(source.to_vec()).ok()?,
String::from_utf8(id.to_vec()).ok()?,
))
}
/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used /// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used
/// by [`compute_card_cid`] and [`compute_yt_video_cid`]. /// by [`compute_card_cid`], [`compute_yt_video_cid`] and
/// [`compute_provider_file_cid`].
fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option<String> { fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option<String> {
if kind.is_empty() || id.is_empty() { if kind.is_empty() || id.is_empty() {
return None; return None;
@@ -1389,3 +1490,84 @@ mod roundtrip_tests {
assert_eq!(decode_url_cid(&card), None); assert_eq!(decode_url_cid(&card), None);
} }
} }
#[cfg(test)]
mod redeem_locator_tests {
use super::*;
/// `docs/cid-formats.md` §8.2 golden vector — moved here from
/// meta-feeder-opensubtitles' local `locator.rs`; the vector proves the
/// move changed no bytes.
#[test]
fn provider_file_matches_the_documented_vector() {
assert_eq!(
compute_provider_file_cid("opensubtitles", "file:7061834").as_deref(),
Some("bagfcaaa2bvxxazloon2we5djorwgk43gnfwgkorxga3dcobtgq")
);
}
#[test]
fn provider_file_round_trips_and_refuses_other_codecs() {
let cid = compute_provider_file_cid("opensubtitles", "file:12658754").unwrap();
assert_eq!(
decode_provider_file_cid(&cid),
Some(("opensubtitles".to_string(), "file:12658754".to_string()))
);
// Identical framing, different codec: a card must not decode as a file.
let card = compute_card_cid("tmdb", "tv:95479").unwrap();
assert_eq!(card, "bagdsaaanar2g2zdcor3duojvgq3ts");
assert_eq!(decode_provider_file_cid(&card), None);
assert_eq!(decode_provider_file_cid("not-a-cid"), None);
assert_eq!(decode_provider_file_cid(""), None);
}
#[test]
fn provider_file_digest_ceiling_is_64_bytes() {
// varint(13) + "opensubtitles" = 14 bytes, so the id may use 50.
assert!(compute_provider_file_cid("opensubtitles", &"x".repeat(50)).is_some());
assert!(compute_provider_file_cid("opensubtitles", &"x".repeat(51)).is_none());
}
#[test]
fn nzb_release_round_trips() {
let cid = compute_nzb_release_cid("api.nzbgeek.info/api", "0123abcdef").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("api.nzbgeek.info/api".to_string(), "0123abcdef".to_string()))
);
// A host:port authority survives too.
let cid = compute_nzb_release_cid("127.0.0.1:8080", "ff00").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("127.0.0.1:8080".to_string(), "ff00".to_string()))
);
}
/// The encoder left-pads an odd-length id, so the decode is the padded form.
#[test]
fn nzb_release_odd_length_id_comes_back_padded() {
let cid = compute_nzb_release_cid("api.nzb.life", "abc").unwrap();
assert_eq!(
decode_nzb_release_cid(&cid),
Some(("api.nzb.life".to_string(), "0abc".to_string()))
);
}
#[test]
fn nzb_release_refuses_other_codecs() {
let card = compute_card_cid("tmdb", "tv:95479").unwrap();
assert_eq!(decode_nzb_release_cid(&card), None);
assert_eq!(decode_nzb_release_cid("bafkreigarbage"), None);
}
#[test]
fn codec_of_reads_the_codec_slot() {
let nzb = compute_nzb_release_cid("api.nzb.life", "ab").unwrap();
assert_eq!(codec_of(&nzb), Some(NZB_RELEASE_CODEC));
let file = compute_provider_file_cid("opensubtitles", "file:1").unwrap();
assert_eq!(codec_of(&file), Some(PROVIDER_FILE_CODEC));
assert_eq!(codec_of(&compute_ipfs_cid(b"hello")), Some(0x55));
assert_eq!(codec_of("QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"), None);
assert_eq!(codec_of(""), None);
}
}
+2
View File
@@ -45,10 +45,12 @@ pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher};
pub use meta_core::FeederStore; pub use meta_core::FeederStore;
pub use plugin::{ pub use plugin::{
upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry,
RedeemClaim,
}; };
pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter};
pub use serve::{ pub use serve::{
configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto,
HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse,
RedeemsResponse,
}; };
pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth};
+94
View File
@@ -17,6 +17,7 @@ use std::path::Path;
use async_trait::async_trait; use async_trait::async_trait;
use futures::stream::BoxStream; use futures::stream::BoxStream;
use serde::{Deserialize, Serialize};
// Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the // Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the
// gateway's original `plugin.rs` re-exported it the same way). // gateway's original `plugin.rs` re-exported it the same way).
@@ -95,6 +96,66 @@ pub enum HashKind {
YtVideo, YtVideo,
} }
/// `RedeemClaim::codec` for a Newznab release locator (`0x1005`).
pub const REDEEM_CODEC_NZB_RELEASE: &str = "nzb-release";
/// `RedeemClaim::codec` for a `provider-file` locator (`0x100A`).
pub const REDEEM_CODEC_PROVIDER_FILE: &str = "provider-file";
/// `RedeemClaim::field` naming the `.nzb` manifest pointer on a release record.
pub const POINTER_FIELD_MANIFEST: &str = "manifest";
/// `RedeemClaim::field` naming the fetched-file pointer on a locator record.
pub const POINTER_FIELD_FILE: &str = "file";
/// A plugin's claim that it can **redeem** a locator family: turn a locator cid
/// into the bytes it names, through `POST /compute` with `record_id` = the
/// locator cid.
///
/// Redeeming usually spends a metered upstream quota (a Newznab `t=get` grab,
/// an OpenSubtitles `/download`), so the gateway routes a redeem only to the
/// plugins claiming the cid's codec, one at a time, and only on a real play. It
/// stores the returned bytes and writes their cid onto the locator record under
/// [`field`](Self::field), so the quota is spent once per file.
///
/// Served at `GET /manifest` (per plugin) and `GET /redeems`. Reflects the
/// plugin's *current* config: a plugin with no credential for a family claims
/// nothing for it.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
pub struct RedeemClaim {
/// Locator family: [`REDEEM_CODEC_NZB_RELEASE`] or [`REDEEM_CODEC_PROVIDER_FILE`].
pub codec: String,
/// Pointer field the gateway writes the redeemed bytes' cid under:
/// [`POINTER_FIELD_MANIFEST`] or [`POINTER_FIELD_FILE`].
pub field: String,
/// Indexer hosts this plugin holds a key for (`nzb-release`) — bare
/// authority, lowercase, never the key itself.
#[serde(default)]
pub hosts: Vec<String>,
/// Provider sources this plugin redeems (`provider-file`), e.g. `opensubtitles`.
#[serde(default)]
pub sources: Vec<String>,
}
impl RedeemClaim {
/// Claim `nzb-release` locators for these indexer hosts.
pub fn nzb_release(hosts: Vec<String>) -> Self {
Self {
codec: REDEEM_CODEC_NZB_RELEASE.to_string(),
field: POINTER_FIELD_MANIFEST.to_string(),
hosts,
sources: Vec::new(),
}
}
/// Claim `provider-file` locators for these sources.
pub fn provider_file(sources: Vec<String>) -> Self {
Self {
codec: REDEEM_CODEC_PROVIDER_FILE.to_string(),
field: POINTER_FIELD_FILE.to_string(),
hosts: Vec::new(),
sources,
}
}
}
/// Startup-only configuration error returned by [`FeederPlugin::configure`]. /// Startup-only configuration error returned by [`FeederPlugin::configure`].
#[derive(Debug, thiserror::Error)] #[derive(Debug, thiserror::Error)]
pub enum ConfigError { pub enum ConfigError {
@@ -215,6 +276,20 @@ pub trait FeederPlugin: Send + Sync + 'static {
&[] &[]
} }
/// Folder under meta-core's `/files/plugin/` where the gateway stores bytes
/// this plugin redeems (e.g. `meta-feeder-usenet`). Stable across
/// deployments — unlike the feeder's compose service name. Default: none.
fn package(&self) -> Option<&'static str> {
None
}
/// Locator families this plugin can redeem through
/// [`compute_outcomes`](FeederPlugin::compute_outcomes) — see
/// [`RedeemClaim`]. Computed from the current config. Default: none.
fn redeems(&self) -> Vec<RedeemClaim> {
Vec::new()
}
/// Self-describing config schema. The feeder SDK serves this at /// Self-describing config schema. The feeder SDK serves this at
/// `GET /config/schema` and renders a generic form from it — the gateway and /// `GET /config/schema` and renders a generic form from it — the gateway and
/// UI carry no per-plugin field knowledge. Default: no configuration. /// UI carry no per-plugin field knowledge. Default: no configuration.
@@ -255,4 +330,23 @@ mod tests {
assert!(msg.contains("scihub")); assert!(msg.contains("scihub"));
assert!(msg.contains("mirrors")); assert!(msg.contains("mirrors"));
} }
/// The JSON keys are the gateway/meta-share contract — pin them.
#[test]
fn redeem_claim_wire_shape() {
let claim = RedeemClaim::nzb_release(vec!["api.nzb.life".into()]);
assert_eq!(
serde_json::to_value(&claim).unwrap(),
serde_json::json!({
"codec": "nzb-release",
"field": "manifest",
"hosts": ["api.nzb.life"],
"sources": []
})
);
// hosts/sources may be omitted on the wire.
let parsed: RedeemClaim =
serde_json::from_str(r#"{"codec":"provider-file","field":"file"}"#).unwrap();
assert_eq!(parsed, RedeemClaim::provider_file(vec![]));
}
} }
+176 -2
View File
@@ -10,6 +10,17 @@
//! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`] //! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`]
//! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404) //! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404)
//! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404) //! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404)
//! - `GET /redeems` → [`RedeemsResponse`] (current redeem claims)
//!
//! ## Redeeming a locator
//!
//! A plugin that advertises a [`RedeemClaim`] answers `POST /compute` with
//! `record_id` = a locator cid of that family (a Newznab `nzb-release`, an
//! OpenSubtitles `provider-file`) by fetching the bytes it names — usually
//! spending a metered quota — and returning ONE `Sha2_256` outcome carrying
//! them. `NotFound` means "not mine" (e.g. no key for that indexer host), so the
//! gateway tries the next claimer. The claims are re-read from `GET /redeems`
//! so a config change shows up without restarting the gateway.
//! //!
//! ## Bytes & the v1 simplification //! ## Bytes & the v1 simplification
//! //!
@@ -21,7 +32,7 @@
//! documented follow-up — those outcomes are metadata-only anyway //! documented follow-up — those outcomes are metadata-only anyway
//! (`bytes: None`), so they never ride this path. //! (`bytes: None`), so they never ride this path.
use std::collections::HashMap; use std::collections::{BTreeMap, HashMap};
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::{Path as FsPath, PathBuf}; use std::path::{Path as FsPath, PathBuf};
use std::sync::Arc; use std::sync::Arc;
@@ -41,7 +52,7 @@ use serde_json::Value;
use tracing::{info, warn}; use tracing::{info, warn};
use crate::config::{self, CONFIG_PAGE_HTML}; use crate::config::{self, CONFIG_PAGE_HTML};
use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome, RedeemClaim};
use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError}; use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError};
use crate::types::{DiscoveryRecord, GatewayError, PluginHealth}; use crate::types::{DiscoveryRecord, GatewayError, PluginHealth};
@@ -94,6 +105,22 @@ pub struct PluginManifest {
pub id: String, pub id: String,
pub served_file_types: Vec<String>, pub served_file_types: Vec<String>,
pub served_content_kinds: Vec<String>, pub served_content_kinds: Vec<String>,
/// [`FeederPlugin::package`] — the `/files/plugin/<package>/` folder redeemed
/// bytes are stored under. `null` when the plugin redeems nothing.
#[serde(default)]
pub package: Option<String>,
/// [`FeederPlugin::redeems`] at the time the manifest was served.
#[serde(default)]
pub redeems: Vec<RedeemClaim>,
}
/// `GET /redeems` — every hosted plugin's current [`RedeemClaim`]s, keyed by
/// `upstream_id` (an empty list for a plugin that claims nothing). Polled by
/// the gateway, so a credential added on the config page is advertised without
/// a gateway restart.
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
pub struct RedeemsResponse {
pub redeems: BTreeMap<String, Vec<RedeemClaim>>,
} }
/// `GET /manifest` — the feeder's hosted plugins + their served types. The /// `GET /manifest` — the feeder's hosted plugins + their served types. The
@@ -224,6 +251,17 @@ impl AppState {
/// status semantics the gateway's `RemoteFeederPlugin` will translate back /// status semantics the gateway's `RemoteFeederPlugin` will translate back
/// into a `GatewayError` on the core side. /// into a `GatewayError` on the core side.
fn gateway_error_response(e: GatewayError) -> Response { fn gateway_error_response(e: GatewayError) -> Response {
// A 429 carries the wait as a `Retry-After` header too, so the gateway can
// forward it to the client (a redeem blocked on a spent quota) instead of
// parsing the message.
if let GatewayError::RateLimited { retry_after_s } = &e {
return (
StatusCode::TOO_MANY_REQUESTS,
[(axum::http::header::RETRY_AFTER, retry_after_s.to_string())],
format!("rate limited; retry in {retry_after_s}s"),
)
.into_response();
}
let (status, msg) = match e { let (status, msg) = match e {
GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()), GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()),
GatewayError::RateLimited { retry_after_s } => ( GatewayError::RateLimited { retry_after_s } => (
@@ -251,6 +289,8 @@ async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
.iter() .iter()
.map(|s| s.to_string()) .map(|s| s.to_string())
.collect(), .collect(),
package: p.package().map(str::to_string),
redeems: p.redeems(),
}) })
.collect(); .collect();
plugins.sort_by(|a, b| a.id.cmp(&b.id)); plugins.sort_by(|a, b| a.id.cmp(&b.id));
@@ -260,6 +300,16 @@ async fn manifest(State(state): State<AppState>) -> Json<ManifestResponse> {
}) })
} }
async fn redeems(State(state): State<AppState>) -> Json<RedeemsResponse> {
Json(RedeemsResponse {
redeems: state
.plugins
.values()
.map(|p| (p.upstream_id().to_string(), p.redeems()))
.collect(),
})
}
async fn health(State(state): State<AppState>) -> Json<HealthResponse> { async fn health(State(state): State<AppState>) -> Json<HealthResponse> {
let mut entries: Vec<PluginHealthEntry> = state let mut entries: Vec<PluginHealthEntry> = state
.plugins .plugins
@@ -492,6 +542,7 @@ pub fn router(
.route("/compute", post(compute)) .route("/compute", post(compute))
.route("/fetch/:upstream_id/:record_id", get(fetch)) .route("/fetch/:upstream_id/:record_id", get(fetch))
.route("/blob/:upstream_id/:cid", get(blob)) .route("/blob/:upstream_id/:cid", get(blob))
.route("/redeems", get(redeems))
.route("/config", get(config_page)) .route("/config", get(config_page))
.route("/config/schema", get(config_schema)) .route("/config/schema", get(config_schema))
.route("/config/values", get(config_values_get).put(config_values_put)) .route("/config/values", get(config_values_get).put(config_values_put))
@@ -541,6 +592,9 @@ struct DegradedPlugin {
reason: String, reason: String,
file_types: &'static [&'static str], file_types: &'static [&'static str],
content_kinds: &'static [&'static str], content_kinds: &'static [&'static str],
/// Preserved so the manifest keeps naming the plugin's folder. A degraded
/// plugin claims no redeems (it cannot redeem anything).
package: Option<&'static str>,
/// Preserved from the wrapped plugin so a degraded/unconfigured feeder still /// Preserved from the wrapped plugin so a degraded/unconfigured feeder still
/// serves its config form — otherwise the dashboard shows "no configuration /// serves its config form — otherwise the dashboard shows "no configuration
/// required" and the operator can never fix the very config that's missing. /// required" and the operator can never fix the very config that's missing.
@@ -586,6 +640,10 @@ impl FeederPlugin for DegradedPlugin {
self.content_kinds self.content_kinds
} }
fn package(&self) -> Option<&'static str> {
self.package
}
fn config_schema(&self) -> crate::config::ConfigSchema { fn config_schema(&self) -> crate::config::ConfigSchema {
self.schema.clone() self.schema.clone()
} }
@@ -608,6 +666,7 @@ pub fn configure_plugins(
// borrow across `configure(&mut self)` / the move into `out`. // borrow across `configure(&mut self)` / the move into `out`.
let file_types = plugin.served_file_types(); let file_types = plugin.served_file_types();
let content_kinds = plugin.served_content_kinds(); let content_kinds = plugin.served_content_kinds();
let package = plugin.package();
// Preserve the config surface so a degraded stand-in still serves the // Preserve the config surface so a degraded stand-in still serves the
// real config form (see DegradedPlugin.schema). // real config form (see DegradedPlugin.schema).
let schema = plugin.config_schema(); let schema = plugin.config_schema();
@@ -627,6 +686,7 @@ pub fn configure_plugins(
reason, reason,
file_types, file_types,
content_kinds, content_kinds,
package,
schema: schema.clone(), schema: schema.clone(),
}) })
}; };
@@ -759,4 +819,118 @@ mod tests {
assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. })); assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. }));
assert!(matches!(events[2], GatewaySearchEvent::Done)); assert!(matches!(events[2], GatewaySearchEvent::Done));
} }
/// A plugin that redeems `nzb-release` locators for one host.
struct StubRedeemer;
#[async_trait]
impl FeederPlugin for StubRedeemer {
fn upstream_id(&self) -> &'static str {
"redeemer"
}
fn configure(&mut self, _cache_dir: &FsPath) -> Result<(), ConfigError> {
Ok(())
}
async fn handle_query(
&self,
_q: &GatewayQuery,
_n: usize,
) -> Result<Vec<DiscoveryRecord>, GatewayError> {
Ok(vec![])
}
async fn compute_outcomes(
&self,
_record_id: &str,
) -> Result<Vec<HashOutcome>, GatewayError> {
Ok(vec![])
}
fn package(&self) -> Option<&'static str> {
Some("meta-feeder-stub")
}
fn redeems(&self) -> Vec<RedeemClaim> {
vec![RedeemClaim::nzb_release(vec!["api.example".into()])]
}
}
async fn serve_stubs() -> String {
let mut plugins: HashMap<String, Arc<dyn FeederPlugin>> = HashMap::new();
plugins.insert("redeemer".to_string(), Arc::new(StubRedeemer));
plugins.insert("stub".to_string(), Arc::new(StubStreaming));
let app = router(plugins, "test".to_string(), std::env::temp_dir());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
format!("http://{addr}")
}
/// The manifest carries `package` + `redeems` per plugin; a plugin that
/// claims nothing serializes `null` + `[]`.
#[tokio::test]
async fn manifest_carries_package_and_redeems() {
let base = serve_stubs().await;
let m: serde_json::Value = reqwest::get(format!("{base}/manifest"))
.await
.unwrap()
.json()
.await
.unwrap();
let redeemer = &m["plugins"][0];
assert_eq!(redeemer["id"], "redeemer");
assert_eq!(redeemer["package"], "meta-feeder-stub");
assert_eq!(
redeemer["redeems"],
serde_json::json!([{
"codec": "nzb-release", "field": "manifest",
"hosts": ["api.example"], "sources": []
}])
);
let stub = &m["plugins"][1];
assert_eq!(stub["package"], serde_json::Value::Null);
assert_eq!(stub["redeems"], serde_json::json!([]));
}
#[tokio::test]
async fn redeems_route_keys_claims_by_upstream_id() {
let base = serve_stubs().await;
let r: RedeemsResponse = reqwest::get(format!("{base}/redeems"))
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(
r.redeems.get("redeemer"),
Some(&vec![RedeemClaim::nzb_release(vec!["api.example".into()])])
);
assert_eq!(r.redeems.get("stub"), Some(&vec![]));
}
/// The gateway forwards a feeder 429's `Retry-After` — pin that it is set.
#[test]
fn rate_limited_sets_retry_after() {
let resp = gateway_error_response(GatewayError::RateLimited { retry_after_s: 3600 });
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
resp.headers()
.get(axum::http::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok()),
Some("3600")
);
// Other errors carry no Retry-After.
let resp = gateway_error_response(GatewayError::NotFound);
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
assert!(resp.headers().get(axum::http::header::RETRY_AFTER).is_none());
}
/// A manifest from a pre-1.3 feeder (no `package` / `redeems`) still parses.
#[test]
fn old_manifest_without_redeem_fields_parses() {
let m: PluginManifest = serde_json::from_str(
r#"{"id":"x","served_file_types":[],"served_content_kinds":[]}"#,
)
.unwrap();
assert!(m.package.is_none() && m.redeems.is_empty());
}
} }