metastar/meta-feeder-gutenberg
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework plugin

Browse Source
This commit is contained in:
Worph
2026-08-24 16:32:04 +02:00
parent a0227f8378
commit 4dc6d46c47
10 changed files with 849 additions and 3525 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/meta-feeder-sdk/src/cache.rs
+16
View File
@@ -53,6 +53,13 @@ const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("s
/// OpenSubtitles search results keyed by `"<tmdb_id>\x01<lang3>"`. Value is the
/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel.
const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles");
/// Small durable plugin scratch values that must survive a restart but aren't
/// content-derived — today, the torznab plugin's Prowlarr indexer-set
/// fingerprint, so a set that changed *while the feeder was down* is still
/// detected as a change on the next boot. Keys are plugin-namespaced
/// (`prowlarr:indexer-set`). redb creates the table on open, so an older
/// `.redb` file gains it transparently.
const MISC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("misc");
/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared
/// via `Arc`).
@@ -105,6 +112,7 @@ impl MidhashCache {
tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?;
tx.open_table(SUBTITLES_TABLE)?;
tx.open_table(OPENSUBTITLES_TABLE)?;
tx.open_table(MISC_TABLE)?;
tx.commit()?;
}
Ok(Self { db: Arc::new(db) })
@@ -112,6 +120,14 @@ impl MidhashCache {
str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE);
str_table_accessors!(
/// Read a durable plugin scratch value (see [`MISC_TABLE`]).
get_misc,
/// Write a durable plugin scratch value (see [`MISC_TABLE`]).
put_misc,
MISC_TABLE,
);
pub fn entry_count(&self) -> Result<u64, redb::Error> {
let tx = self.db.begin_read()?;
let table = tx.open_table(MIDHASH_TABLE)?;
crates/meta-feeder-sdk/src/config_page.html
+42 -19
View File
@@ -17,6 +17,7 @@
.field { margin: 0 0 16px; }
.field > label { display: block; font-weight: 600; margin: 0 0 4px; }
.help { color: #8a8a96; font-size: 12px; margin: 3px 0 0; }
.req { color: #f0a0a0; margin-left: 3px; font-weight: 700; }
input[type=text], input[type=password], input[type=number], textarea {
width: 100%; padding: 8px 10px; border-radius: 8px;
border: 1px solid #2a2a35; background: #15151c; color: #e6e6ea; font: inherit;
@@ -46,7 +47,7 @@
</head>
<body>
<h1 id="title">Plugin configuration</h1>
<p class="sub">Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect.</p>
<p class="sub">Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked <span class="req">*</span> are required.</p>
<div id="banner" class="banner"></div>
<form id="form"></form>
<div class="bar" id="bar" style="display:none">
@@ -102,15 +103,24 @@ function linkify(text) {
function helpEl(f) { return f.help ? el("p", { class: "help" }, ...linkify(f.help)) : null; }
// A "*" marker for required fields (null for optional ones).
function reqStar(f) { return f.required ? el("span", { class: "req", text: "*", title: "required" }) : null; }
// Render one scalar field. `vals` is the object holding the value/`_set` flag.
// Every returned widget exposes `_collect()` and `_key` so collect() can read it.
function fieldInput(f, vals) {
if (f.type === "bool") {
const cb = el("input", { type: "checkbox" });
cb.checked = vals[f.key] === true;
cb._get = () => cb.checked;
return el("label", {}, cb, " ", el("span", { text: " " + f.label }));
const w = el("div", { class: "field" },
el("label", {}, cb, " ", el("span", { text: " " + f.label }), reqStar(f)),
helpEl(f));
w._collect = () => cb.checked;
w._key = f.key;
return w;
}
const label = el("label", { text: f.label });
const star = reqStar(f); if (star) label.append(star);
let input;
if (f.type === "secret") {
const isSet = vals[f.key + "_set"] === true;
@@ -134,14 +144,6 @@ function fieldInput(f, vals) {
const wrap = el("div", { class: "field" }, label, input, helpEl(f));
wrap._collect = () => input._get();
wrap._key = f.key;
// bool path returns a <label>; normalize collect/key for it
if (f.type === "bool") {
const w = el("div", { class: "field" }, label, helpEl(f));
const cb = label.querySelector("input");
w._collect = () => cb.checked;
w._key = f.key;
return w;
}
return wrap;
}
@@ -175,8 +177,9 @@ function recordArray(f, vals) {
const add = el("button", { type: "button", class: "add", text: "+ add " + f.label.toLowerCase().replace(/s$/, "") });
add.onclick = () => addRecord({});
const wrap = el("div", { class: "field" },
el("label", { text: f.label }), helpEl(f), list, add);
const lbl = el("label", { text: f.label });
const star = reqStar(f); if (star) lbl.append(star);
const wrap = el("div", { class: "field" }, lbl, helpEl(f), list, add);
wrap._key = f.key;
wrap._collect = () => Array.from(list.children).map((c) => c._collect());
return wrap;
@@ -215,14 +218,34 @@ async function save() {
headers: { "Content-Type": "application/json" },
body: JSON.stringify(collect()),
});
if (!res.ok) throw new Error((await res.text()) || res.status);
if (!res.ok) {
const detail = (await res.text().catch(() => "")).trim();
throw new Error(detail ? `the feeder rejected it (HTTP ${res.status}): ${detail}`
: `the feeder returned HTTP ${res.status}`);
}
const j = await res.json().catch(() => ({}));
if (j.restart_required) banner("warn", "Saved. Restart the feeder container for changes to take effect.");
else banner("ok", "Saved.");
// Re-fetch so secret pills reflect the new stored state.
await load();
if (j.reloading) {
// The feeder is restarting itself (~2-3s) to apply the change. Poll it
// back up, then refresh the form so secret pills reflect stored state.
banner("warn", "Saved — reloading the feeder to apply (~3 s)…");
let tries = 6;
const poll = async () => {
try { await load(); banner("ok", "Reloaded — new config applied."); }
catch { if (--tries > 0) setTimeout(poll, 1500); else banner("warn", "Saved & reloading — reopen this panel shortly to confirm."); }
};
setTimeout(poll, 3000);
} else {
if (j.restart_required) banner("warn", "Saved. Restart the feeder container for changes to take effect.");
else banner("ok", "Saved.");
// Re-fetch so secret pills reflect the new stored state.
await load();
}
} catch (e) {
banner("err", "Save failed: " + e.message);
const msg = (e && e.message) ? e.message : String(e);
const friendly = /Failed to fetch|NetworkError|load failed/i.test(msg)
? "couldn't reach the feeder — it may be restarting; wait a moment and try again."
: msg;
banner("err", "Could not save: " + friendly);
} finally {
$("save").disabled = false;
}
crates/meta-feeder-sdk/src/enrich.rs
+31 -7
View File
@@ -73,16 +73,40 @@ fn env_nonempty(key: &str) -> Option<String> {
.filter(|s| !s.is_empty())
}
/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the
/// app-compose convention `<prefix>-feeder` → `<prefix>-<plugin>:8080`
/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets
/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit
/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns
/// `None` when `HOSTNAME` is unset or not in `<prefix>-feeder` form, in which case
/// the caller falls back to the env seed (then soft-skips that plugin).
fn derive_sidecar_url(plugin: &str) -> Option<String> {
let host = env_nonempty("HOSTNAME")?;
let prefix = host.strip_suffix("-feeder")?;
Some(format!("http://{prefix}-{plugin}:8080"))
}
impl EnrichmentConfig {
/// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset —
/// the feeder then falls back to the legacy "return record, gateway stores"
/// path (backward compatible; used by tests and gateway-stores deployments).
/// the **bundled-feeder** path (records returned over HTTP, the gateway
/// dispatcher stores them). Self-publishing feeders that source meta-core from
/// their own dashboard config use [`from_meta_core`](Self::from_meta_core).
pub fn from_env() -> Option<Self> {
let meta_core_url = env_nonempty("META_CORE_URL")?;
Some(EnrichmentConfig {
Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?))
}
/// Build with an explicit meta-core root — sourced from the feeder's own
/// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The
/// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the
/// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest
/// keep their env seeds / defaults. Secrets (tmdb token, language) are
/// overlaid by the caller from the per-feeder config.
pub fn from_meta_core(meta_core_url: String) -> Self {
EnrichmentConfig {
meta_core_url,
filename_parser_url: env_nonempty("FILENAME_PARSER_URL"),
tmdb_url: env_nonempty("TMDB_PLUGIN_URL"),
filename_parser_url: env_nonempty("FILENAME_PARSER_URL")
.or_else(|| derive_sidecar_url("filename-parser")),
tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")),
tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")),
tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()),
gateway_peer: env_nonempty("META_GATEWAY_PEER_ID")
@@ -96,7 +120,7 @@ impl EnrichmentConfig {
opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"),
opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES")
.unwrap_or_else(|| "en".to_string()),
})
}
}
}
crates/meta-feeder-sdk/src/filename_meta.rs
+27 -23
View File
@@ -57,9 +57,9 @@ pub struct SeasonEpisode {
/// is deliberately **not** given a single `episode`.
pub is_pack: bool,
/// True when [`season`] came from an explicit season-bearing token
/// (`S01E05`, `Season 2`, `S03`), as opposed to the "episode with no
/// season → assume 1" default. Lets a per-file reparse know whether to
/// borrow the season from the pack/parent title (see
/// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left
/// season-less (no "assume season 1" default), so a per-file reparse uses
/// this to know it should borrow the season from the pack/parent title (see
/// [`extract_season_episode_with_context`]).
pub season_explicit: bool,
}
@@ -550,21 +550,22 @@ pub fn extract_season_episode(title: &str) -> SeasonEpisode {
let mut out = SeasonEpisode::default();
// season/episode 0 are valid; only < 0 means "not found".
let mut sea_eff = sea;
if sea >= 0.0 {
out.season = Some(fmt_num(sea));
out.season_explicit = true;
}
if ep >= 0.0 {
out.episode = Some(fmt_num(ep));
if sea_eff < 0.0 {
// An episode with no season — assume season 1 (matches TS). The
// season is *not* explicit, so a per-file reparse may override it
// from the parent (pack) title.
sea_eff = 1.0;
out.season = Some("1".to_string());
// An episode with no season is left season-LESS on purpose: a season is
// only asserted when explicitly hinted — an S-token in the title here,
// or (via extract_season_episode_with_context) the parent/pack title.
// Absolute-numbered releases ("Show - 28") thus carry an episode and no
// season, so the UI buckets them under "Episodes" rather than a
// fabricated "Season 1". `increment` encodes a season, so it is only
// emitted when the season is known.
if sea >= 0.0 {
out.increment = Some(fmt_num(sea * 10000.0 + ep));
}
out.increment = Some(fmt_num(sea_eff * 10000.0 + ep));
} else if sea >= 0.0 {
// Season present, no episode → a whole-season pack (`Season 3`, `S03`).
out.is_pack = true;
@@ -583,8 +584,8 @@ pub fn extract_season_episode(title: &str) -> SeasonEpisode {
pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode {
let mut file_se = extract_season_episode(filename);
// Only borrow the parent's season for a concrete single episode whose own
// filename carries no *explicit* season token (so its season is the assumed
// `1` default). A per-file record is an episode, not a pack, so we don't
// filename carries no *explicit* season token (so it is otherwise
// season-less). A per-file record is an episode, not a pack, so we don't
// propagate the parent's pack/range shape here.
if file_se.episode.is_some() && !file_se.season_explicit {
if let Some(s) = find_pack_season(&preclean(parent)) {
@@ -1004,12 +1005,13 @@ mod tests {
}
#[test]
fn anime_trailing_number_assumes_season_one() {
// No season marker → episode via rightmost-number fallback,
// season assumed 1. SFV id stripped, brackets ignored.
fn anime_trailing_number_is_season_less() {
// No season marker → episode via rightmost-number fallback, and NO
// season is fabricated (absolute numbering). SFV id stripped, brackets
// ignored. The UI buckets a season-less episode under "Episodes".
assert_eq!(
se("[SubsPlease] Some Anime - 117 [FC412C51]"),
(Some("1".into()), Some("117".into()))
(None, Some("117".into()))
);
}
@@ -1184,8 +1186,8 @@ mod tests {
#[test]
fn per_file_context_recovers_season_from_parent() {
// Opened pack: the inner file carries the episode, the pack title the
// season. `Frieren - 03.mkv` alone assumes S1; with the S2 pack parent
// it resolves to S2E3.
// season. `Frieren - 03.mkv` alone is season-less; with the S2 pack
// parent it resolves to S2E3 (the parent is the "season hint").
let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv");
assert_eq!(r.season.as_deref(), Some("2"));
assert_eq!(r.episode.as_deref(), Some("3"));
@@ -1198,12 +1200,14 @@ mod tests {
}
#[test]
fn absolute_numbered_anime_still_assumes_season_one() {
// No season keyword → the rightmost-number fallback still fires, season
// assumed 1 (Phase 3 remaps this against TMDB episode counts later).
fn absolute_numbered_anime_is_season_less() {
// No season keyword → the rightmost-number fallback still finds the
// episode, but NO season is fabricated. Season is only ever asserted
// from an explicit token (title or parent), never inferred — so an
// absolute-numbered release lands in the UI's "Episodes" bucket.
assert_eq!(
se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"),
(Some("1".into()), Some("37".into()))
(None, Some("37".into()))
);
}
crates/meta-feeder-sdk/src/hash.rs
+555
View File
@@ -160,6 +160,333 @@ fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String {
format!("b{}", base32_lower_no_padding(&wire))
}
/// Custom multicodec for a self-describing **Newznab release** locator
/// (`nzb-release`). MetaMesh-private, adjacent to the torrent-file codecs in the
/// `0x10xx` range. Unlike a content hash, the cid *embeds* the indexer host +
/// release id in an identity multihash, so the credentialed meta-share peer
/// decodes `{host,id}` straight from the cid and grabs the `.nzb` via `t=get`
/// only at playback — no side-table, no `.nzb` download at search time. (The
/// `0x1004` `sha256(host‖id)` opaque hash and the `0x1003` `nzb-posting`
/// variant were both removed; only this self-describing form is emitted.)
pub const NZB_RELEASE_CODEC: u64 = 0x1005;
/// Digest byte ceiling for an `nzb-release` cid, matching meta-share's
/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash.
const NZB_RELEASE_MAX_DIGEST: usize = 64;
/// Encode a self-describing `nzb-release` CID from a Newznab indexer's host +
/// bare release id (the hex `<guid>` id). The multihash is *identity* (`0x00`)
/// and its digest embeds the locator as `varint(host_len) ‖ host ‖ id_bytes`,
/// where `id_bytes` is the hex-decoded id. meta-share's `decode_nzb_release_cid`
/// is the exact inverse. Host-namespaced: the same release on two indexers
/// yields different cids.
///
/// Returns `None` when the id isn't hex or the digest would overflow the
/// multihash budget (an oversized host) — the caller drops that row.
pub fn compute_nzb_release_cid(api_base: &str, release_id: &str) -> Option<String> {
let id_bytes = decode_hex_id(release_id)?;
let host_b = api_base.as_bytes();
// identity-multihash digest = varint(host_len) ‖ host ‖ id_bytes
let mut digest = Vec::with_capacity(2 + host_b.len() + id_bytes.len());
write_pb_varint(host_b.len() as u64, &mut digest);
digest.extend_from_slice(host_b);
digest.extend_from_slice(&id_bytes);
if digest.len() > NZB_RELEASE_MAX_DIGEST {
return None;
}
// CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest]
let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
wire.push(0x01);
write_pb_varint(NZB_RELEASE_CODEC, &mut wire);
wire.push(0x00); // multihash code: identity
write_pb_varint(digest.len() as u64, &mut wire);
wire.extend_from_slice(&digest);
Some(format!("b{}", base32_lower_no_padding(&wire)))
}
/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the
/// id is all-hex by construction; odd-length ids are left-padded with a `0`
/// nibble. Returns `None` on any non-hex byte (defensive).
fn decode_hex_id(id: &str) -> Option<Vec<u8>> {
let padded;
let s: &str = if id.len() % 2 == 1 {
padded = format!("0{id}");
&padded
} else {
id
};
let bytes = s.as_bytes();
let mut out = Vec::with_capacity(bytes.len() / 2);
for pair in bytes.chunks(2) {
let hi = (pair[0] as char).to_digit(16)?;
let lo = (pair[1] as char).to_digit(16)?;
out.push(((hi << 4) | lo) as u8);
}
Some(out)
}
/// Custom multicodec for a **card locator** — the identity of a *work* (a
/// series, a film) as published by a metadata bridge, rather than of any
/// bytes. MetaMesh-private, the next free slot after `0x1006` url. (`0x1004`
/// nzb-release-hash stays **retired — do not reuse**. `0x1003` was retired too,
/// but has since been **revived** for the self-scanned Usenet posting identity —
/// see [`NZB_POSTING_CODEC`] and `METADATA_KEYS.md` §2.)
///
/// This is the first CID family in the platform where **no bytes exist
/// anywhere, ever** — `0x1005`/`0x1006` are locators too, but both eventually
/// resolve to bytes (NNTP, HTTP), and `0x1000` midhash256 is a digest over
/// real file bytes. A card is pure identity: the record *is* the payload, and
/// it lives in meta-core. Consequences, all already handled:
///
/// * Bitswap seeding is gated on `HashKind::Sha2_256` (gateway invariant 6),
/// so a card is never offered to the swarm — nothing to seed.
/// * meta-share's byte path 404s for it, which is correct.
/// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for
/// `canonical_cid` — which is what makes the future "one card, two CIDs"
/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe.
pub const CARD_LOCATOR_CODEC: u64 = 0x1007;
/// Digest byte ceiling for a card locator, matching meta-share's
/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash.
const CARD_LOCATOR_MAX_DIGEST: usize = 64;
/// Encode a self-describing **card locator** CID from a metadata source and
/// that source's own id for the work (`("tmdb", "tv:95479")`,
/// `("mal", "52991")`). The multihash is *identity* (`0x00`) and its digest
/// embeds the locator as `varint(source_len) ‖ source ‖ id`, both UTF-8 —
/// exactly the shape [`compute_nzb_release_cid`] uses, minus the hex decode
/// (a card id is arbitrary text, not a hex release id).
///
/// **The CID is a pure function of `(source, id)`**, which is the whole point:
/// any peer that knows a tmdb id can derive the card's address offline and
/// look it up locally, with no discovery round-trip. It also means the same
/// card discovered by two different peers converges on one meta-core record
/// for free, via the `cids/<bareCid>` reverse index.
///
/// Source-namespaced: the same work on TMDB and on MyAnimeList yields
/// different cids. Merging those two into one record is a meta-core alias
/// operation driven by an explicit cross-source id mapping — never inferred
/// here (see `meta-gateway/docs/others/card-tier-search.md` §9).
///
/// Returns `None` when `source` is empty or the digest would overflow the
/// multihash budget — the caller drops that card.
pub fn compute_card_cid(source: &str, id: &str) -> Option<String> {
if source.is_empty() || id.is_empty() {
return None;
}
let source_b = source.as_bytes();
let id_b = id.as_bytes();
// identity-multihash digest = varint(source_len) ‖ source ‖ id
let mut digest = Vec::with_capacity(2 + source_b.len() + id_b.len());
write_pb_varint(source_b.len() as u64, &mut digest);
digest.extend_from_slice(source_b);
digest.extend_from_slice(id_b);
if digest.len() > CARD_LOCATOR_MAX_DIGEST {
return None;
}
// CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest]
let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
wire.push(0x01);
write_pb_varint(CARD_LOCATOR_CODEC, &mut wire);
wire.push(0x00); // multihash code: identity
write_pb_varint(digest.len() as u64, &mut wire);
wire.extend_from_slice(&digest);
Some(format!("b{}", base32_lower_no_padding(&wire)))
}
// ---------------------------------------------------------------------------
// Delegated-playback locators — `yt-video` (0x1008) and `ext-play` (0x1009)
// ---------------------------------------------------------------------------
//
// These two are the limit case *beyond* `card` (0x1007). `card` addresses a
// work with no bytes anywhere; these address a **rendition whose bytes exist
// but are permanently someone else's**. An external player renders the media,
// nothing is transported, nothing is content-addressed, nothing is re-seedable.
//
// See `docs/cid-formats.md` §7 and
// `docs/study/listenbrainz-youtube-playback-tier-2026-08-21.md`.
/// Custom multicodec for a **YouTube rendition** — delegated playback.
///
/// Its own codec rather than a `url` (`0x1006`) for three reasons, in
/// descending order of how much trouble the alternative causes:
///
/// * **`0x1006` means "fetch these bytes once and seed them".** Point a `url`
/// locator at a `watch` page and meta-share does exactly that: fetches the
/// HTML, chunks it, seeds it, and links the resulting content CID back onto
/// the record *as the file*. That is the MSR1 poisoning shape, except
/// permanent and replicated across peers. A resolved `googlevideo` URL is no
/// better — time-limited and IP-bound, so it is not a locator at all.
/// * **The id is canonical, a URL is not.** `youtube.com` / `music.youtube.com`
/// / `youtu.be` all address the same video, and query parameters are tracking
/// noise, so the same rendition would mint several different CIDs. Being a
/// pure function of `(kind, id)`, this codec makes two peers that resolve the
/// same track converge on one meta-core record for free, via the
/// `cids/<bareCid>` reverse index — the property that makes [`compute_card_cid`]
/// useful, for the same reason.
/// * **It is tiny** — 11 bytes of id plus a short kind prefix.
///
/// ⚠ **An immutable id is not durable playback.** Takedowns, geo-blocks and
/// embed-disabled uploads all kill a reference whose CID stays valid forever —
/// the same rot class as [`NZB_RELEASE_CODEC`]. The mitigation is at the
/// *record* level (carry several ranked references per recording so a client
/// can fall through), never at the CID level.
pub const YT_VIDEO_CODEC: u64 = 0x1008;
/// Custom multicodec for an **external page to open, never fetch**.
///
/// Wire-identical to `url` (`0x1006`) apart from the codec, and that difference
/// is the entire contract: `0x1006` resolves cache-through and lands in the
/// blockstore, `0x1009` **must be refused by the byte path** and only ever
/// surfaces as an "Open on …↗" affordance. Use it for anything worth linking
/// but not worth a per-provider codec (Bandcamp, Jamendo, an arbitrary stream
/// page).
pub const EXT_PLAY_CODEC: u64 = 0x1009;
/// Encode a [`YT_VIDEO_CODEC`] locator from a `kind` and a bare YouTube id.
///
/// `kind` is `"video"` | `"playlist"` | `"channel"`; `id` is the bare
/// identifier (`4D7u5KF7SP8`, `OLAK5uy_…`, `UCRr1xG_2WIDs18a6cIiCxeA`) — **not**
/// a URL, and not a `watch?v=` fragment. The framing is byte-identical to
/// [`compute_card_cid`] on purpose, so each of the seven rank/decode
/// implementations extends a parser it already wrote rather than learning a
/// novel shape.
///
/// Returns `None` on an empty field or a digest past the multihash budget —
/// the caller drops that reference.
pub fn compute_yt_video_cid(kind: &str, id: &str) -> Option<String> {
kinded_locator_cid(YT_VIDEO_CODEC, kind, id)
}
/// Encode an [`EXT_PLAY_CODEC`] locator from an `http(s)` URL.
///
/// The scheme check is not cosmetic: it is what keeps a `javascript:` or
/// `data:` payload from ever reaching a client that will hand this string to
/// an anchor or a new tab.
pub fn compute_ext_play_cid(url: &str) -> Option<String> {
if !(url.starts_with("http://") || url.starts_with("https://")) {
return None;
}
let digest = url.as_bytes();
if digest.is_empty() || digest.len() > CARD_LOCATOR_MAX_DIGEST {
return None;
}
Some(identity_locator_cid(EXT_PLAY_CODEC, digest))
}
/// Shared encoder for the `varint(len(kind)) ‖ kind ‖ id` locator framing used
/// by [`compute_card_cid`] and [`compute_yt_video_cid`].
fn kinded_locator_cid(codec: u64, kind: &str, id: &str) -> Option<String> {
if kind.is_empty() || id.is_empty() {
return None;
}
let kind_b = kind.as_bytes();
let id_b = id.as_bytes();
let mut digest = Vec::with_capacity(2 + kind_b.len() + id_b.len());
write_pb_varint(kind_b.len() as u64, &mut digest);
digest.extend_from_slice(kind_b);
digest.extend_from_slice(id_b);
if digest.len() > CARD_LOCATOR_MAX_DIGEST {
return None;
}
Some(identity_locator_cid(codec, &digest))
}
/// CIDv1 with an *identity* multihash:
/// `[version=0x01][codec varint][mh=0x00][len varint][digest]`, base32lower.
fn identity_locator_cid(codec: u64, digest: &[u8]) -> String {
let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
wire.push(0x01);
write_pb_varint(codec, &mut wire);
wire.push(0x00); // multihash code: identity
write_pb_varint(digest.len() as u64, &mut wire);
wire.extend_from_slice(digest);
format!("b{}", base32_lower_no_padding(&wire))
}
/// Custom multicodec for a **Usenet posting** identity (`nzb-posting`),
/// minted from the article Message-IDs of a release we scanned ourselves.
///
/// Unlike [`NZB_RELEASE_CODEC`] (`0x1005`), this is **not** a locator: it
/// embeds no indexer host, so any peer with a plain NNTP provider can fetch the
/// articles — no `IndexerCred`, no `t=get` grab. That portability is the whole
/// point of running our own header scanner.
///
/// It is also **not** fetchable by cid: the digest is over the Message-ID set,
/// not over any block's bytes, so bitswap can never satisfy a want for it (the
/// receiver derives a block's cid by hashing the block — see
/// `beetswap::incoming_stream`). The `.nzb` manifest travels separately, as an
/// ordinary sha2-256 IPFS cid in the record's `manifest` field. See
/// `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5.
pub const NZB_POSTING_CODEC: u64 = 0x1003;
/// Mint an `nzb-posting` cid (`0x1003`) from a release's article Message-IDs.
///
/// # The normalisation rule IS the dedup contract
///
/// Mirrored verbatim in meta-share (`nzb/manifest.rs`), which re-derives this
/// digest from the fetched manifest and rejects a mismatch. **Any change here
/// must land on both sides together** — a drift silently makes every posting
/// unplayable. The rule:
///
/// 1. Trim each id, and strip a surrounding `<…>` pair if present (NZB
/// `<segment>` bodies carry the bare form, but be defensive — NNTP `BODY`
/// re-adds the brackets).
/// 2. Drop empties.
/// 3. Deduplicate, then **sort** lexicographically by bytes.
/// 4. Join with `\n` and SHA-256 the result.
///
/// Case is **preserved**: a Message-ID's local part is case-sensitive per RFC
/// 5322, so lowercasing could collide two genuinely distinct articles.
///
/// Sorting is what makes the cid survive segment/file reordering between two
/// NZBs of the same posting — the reason we hash the *id set* and never the raw
/// `.nzb` bytes (the retired `0x1004` did the latter and broke dedup; see
/// meta-share `nzb/mod.rs`).
///
/// **v1 scope:** every segment counts, par2 volumes included. Two NZBs for one
/// posting that disagree about whether to bundle par2 therefore mint different
/// cids. Acceptable while we are the only producer of these manifests; if a
/// second producer ever appears, the fix is to filter par2 subjects here *and*
/// in meta-share's mirror — do not do it on one side only.
pub fn compute_nzb_posting_cid<S: AsRef<str>>(message_ids: &[S]) -> String {
let mut ids: Vec<&str> = message_ids
.iter()
.map(|s| {
let t = s.as_ref().trim();
t.strip_prefix('<')
.and_then(|r| r.strip_suffix('>'))
.unwrap_or(t)
})
.filter(|s| !s.is_empty())
.collect();
ids.sort_unstable();
ids.dedup();
let mut hasher = Sha256::new();
for (i, id) in ids.iter().enumerate() {
if i > 0 {
hasher.update(b"\n");
}
hasher.update(id.as_bytes());
}
let digest: [u8; 32] = hasher.finalize().into();
// CIDv1: [version=0x01][codec varint][mh code=0x12 sha2-256][len=32][digest]
let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len());
wire.push(0x01);
write_pb_varint(NZB_POSTING_CODEC, &mut wire);
wire.push(0x12); // multihash code: sha2-256
write_pb_varint(digest.len() as u64, &mut wire);
wire.extend_from_slice(&digest);
format!("b{}", base32_lower_no_padding(&wire))
}
/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's
/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144
/// --hash=sha2-256 <file>`.
@@ -456,6 +783,82 @@ mod tests {
assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi");
}
#[test]
fn yt_video_cid_matches_golden_vector() {
// Pinned by `/cid-rank-vectors.json`'s `yt-video-locator` entry. The id
// is the real resolver output for Daft Punk / "Get Lucky" measured in
// the delegated-playback study §4.3.
assert_eq!(
compute_yt_video_cid("video", "4D7u5KF7SP8").unwrap(),
"bagecaaarav3gszdfn42ein3vgvfumn2tka4a"
);
}
#[test]
fn yt_video_cid_framing_matches_card_locator() {
// The two codecs share the `varint(len(kind)) || kind || id` framing on
// purpose, so every decoder extends a parser it already wrote. If this
// ever fails, the two shapes have drifted and §7.2 of docs/cid-formats.md
// is no longer true.
let yt = compute_yt_video_cid("tmdb", "tv:95479").unwrap();
let card = compute_card_cid("tmdb", "tv:95479").unwrap();
// Same digest, different codec => same length, differing only in the
// codec varint region.
assert_ne!(yt, card);
assert_eq!(yt.len(), card.len());
}
#[test]
fn yt_video_cid_is_a_pure_function_of_kind_and_id() {
// Convergence is the reason this is a codec and not a URL: two peers
// that resolve the same video must derive the same address offline.
assert_eq!(
compute_yt_video_cid("video", "4D7u5KF7SP8"),
compute_yt_video_cid("video", "4D7u5KF7SP8")
);
// ...and the kind is part of the identity, not decoration.
assert_ne!(
compute_yt_video_cid("video", "OLAK5uy_kZ8Xq"),
compute_yt_video_cid("playlist", "OLAK5uy_kZ8Xq")
);
}
#[test]
fn yt_video_cid_rejects_empty_fields() {
assert!(compute_yt_video_cid("", "4D7u5KF7SP8").is_none());
assert!(compute_yt_video_cid("video", "").is_none());
}
#[test]
fn ext_play_cid_matches_golden_vector() {
assert_eq!(
compute_ext_play_cid("https://example.bandcamp.com/album/x").unwrap(),
"bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa"
);
}
#[test]
fn ext_play_cid_rejects_non_http_schemes() {
// Load-bearing: a client hands this string to an anchor or a new tab,
// so a `javascript:` or `data:` payload must never mint a locator.
assert!(compute_ext_play_cid("javascript:alert(1)").is_none());
assert!(compute_ext_play_cid("data:text/html,<script>").is_none());
assert!(compute_ext_play_cid("ftp://example.com/x").is_none());
assert!(compute_ext_play_cid("").is_none());
}
#[test]
fn ext_play_cid_differs_from_the_url_locator_for_the_same_url() {
// The whole contract lives in the codec slot: `0x1006` means "fetch and
// seed these bytes", `0x1009` means "open this, there are no bytes for
// us". Wire-identical otherwise — which is exactly why a decoder that
// matched on *shape* would seed a Bandcamp page as the file.
let url = "https://example.bandcamp.com/album/x";
let ext = compute_ext_play_cid(url).unwrap();
let card = compute_card_cid("x", url).unwrap();
assert_ne!(ext, card);
}
#[test]
fn ipfs_cid_single_leaf_matches_kubo() {
assert_eq!(
@@ -518,4 +921,156 @@ mod tests {
assert_eq!(out.blocks[0].1.len(), 0);
assert_eq!(out.blocks[0].0, out.root);
}
// ---- card locator (0x1007) --------------------------------------------
/// Inverse of [`base32_lower_no_padding`], test-only. Lets the card tests
/// assert the actual CIDv1 bytes (codec, multihash code, digest) rather
/// than just round-tripping the encoder against itself.
fn base32_lower_decode(s: &str) -> Vec<u8> {
const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567";
let mut out = Vec::new();
let (mut acc, mut bits) = (0u32, 0u32);
for c in s.bytes() {
let v = ALPHABET.iter().position(|&a| a == c).expect("base32 char") as u32;
acc = (acc << 5) | v;
bits += 5;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
out
}
/// The property the whole card design leans on: the CID is a pure function
/// of `(source, id)`, so any peer can derive a card's address offline. If
/// this literal ever changes, every already-published card silently forks
/// into a second identity — treat a failure here as a wire break.
#[test]
fn card_cid_is_deterministic() {
let cid = compute_card_cid("tmdb", "tv:95479").expect("card cid");
assert_eq!(cid, compute_card_cid("tmdb", "tv:95479").unwrap());
assert_eq!(cid, "bagdsaaanar2g2zdcor3duojvgq3ts");
}
/// Source- and kind-namespaced: the same numeric id on a different bridge,
/// or the same id under a different media kind, must not collide.
#[test]
fn card_cid_is_source_and_kind_namespaced() {
let tmdb = compute_card_cid("tmdb", "tv:95479").unwrap();
let mal = compute_card_cid("mal", "95479").unwrap();
let movie = compute_card_cid("tmdb", "movie:95479").unwrap();
assert_ne!(tmdb, mal);
assert_ne!(tmdb, movie);
}
/// Pins the on-the-wire CIDv1 layout: v1, codec `0x1007` (varint
/// `[0x87,0x20]`), **identity** multihash `0x00` (this family has no
/// digest — the locator itself is the payload), then
/// `varint(source_len) ‖ source ‖ id`.
#[test]
fn card_cid_wire_shape() {
let cid = compute_card_cid("tmdb", "tv:95479").unwrap();
assert!(cid.starts_with('b'), "multibase base32-lower prefix");
let wire = base32_lower_decode(&cid[1..]);
assert_eq!(wire[0], 0x01, "CIDv1");
assert_eq!(&wire[1..3], &[0x87, 0x20], "codec varint for 0x1007");
assert_eq!(wire[3], 0x00, "identity multihash — no digest");
let digest_len = wire[4] as usize;
let digest = &wire[5..5 + digest_len];
assert_eq!(digest[0], 4, "varint(len(\"tmdb\"))");
assert_eq!(&digest[1..5], b"tmdb");
assert_eq!(&digest[5..], b"tv:95479");
}
/// Empty inputs are rejected rather than encoded as an ambiguous locator,
/// and an id past the 64-byte multihash budget (meta-share's
/// `CidGeneric<64>`) returns `None` so the caller drops the card instead of
/// emitting a cid that peer would refuse to parse.
#[test]
fn card_cid_rejects_empty_and_oversized() {
assert!(compute_card_cid("", "tv:1").is_none());
assert!(compute_card_cid("tmdb", "").is_none());
let huge = "x".repeat(CARD_LOCATOR_MAX_DIGEST);
assert!(compute_card_cid("tmdb", &huge).is_none());
// Exactly at the ceiling still encodes.
let fits = "x".repeat(CARD_LOCATOR_MAX_DIGEST - 5); // varint(4) + "tmdb"
assert!(compute_card_cid("tmdb", &fits).is_some());
}
// ---- nzb-posting (0x1003) — the dedup contract -----------------------
//
// These pin the normalisation rule documented on `compute_nzb_posting_cid`.
// meta-share re-derives the same digest from the fetched manifest and
// rejects a mismatch, so a change that breaks one of these silently makes
// every self-scanned posting unplayable. Mirror any edit on both sides.
/// Segment/file **order must not matter** — that is the whole reason we
/// hash a sorted id set instead of the raw `.nzb` bytes (the retired
/// `0x1004` did the latter and broke dedup across reposts).
#[test]
fn nzb_posting_cid_is_order_independent() {
let a = ["[email protected]", "[email protected]", "[email protected]"];
let b = ["[email protected]", "[email protected]", "[email protected]"];
let c = ["[email protected]", "[email protected]", "[email protected]"];
assert_eq!(compute_nzb_posting_cid(&a), compute_nzb_posting_cid(&b));
assert_eq!(compute_nzb_posting_cid(&a), compute_nzb_posting_cid(&c));
}
/// Angle brackets, surrounding whitespace, empty entries and duplicates are
/// all normalised away. NZB `<segment>` bodies carry the bare form but NNTP
/// `BODY` re-adds the brackets, so both spellings must converge.
#[test]
fn nzb_posting_cid_normalises_brackets_blanks_and_dupes() {
let bare = ["[email protected]", "[email protected]"];
let noisy = [" <[email protected]> ", "[email protected]", "", " ", "[email protected]"];
assert_eq!(
compute_nzb_posting_cid(&bare),
compute_nzb_posting_cid(&noisy)
);
}
/// Case is preserved: a Message-ID's local part is case-sensitive per RFC
/// 5322, so lowercasing could collide two genuinely distinct articles.
#[test]
fn nzb_posting_cid_is_case_sensitive() {
assert_ne!(
compute_nzb_posting_cid(&["[email protected]"]),
compute_nzb_posting_cid(&["[email protected]"])
);
}
/// A different article set is a different posting.
#[test]
fn nzb_posting_cid_differs_on_different_ids() {
assert_ne!(
compute_nzb_posting_cid(&["[email protected]", "[email protected]"]),
compute_nzb_posting_cid(&["[email protected]", "[email protected]"])
);
}
/// Wire shape: CIDv1, codec `0x1003` (varint `[0x83, 0x20]`), multihash
/// sha2-256 (`0x12`), 32-byte digest. Pinned because meta-share matches on
/// the **codec** slot — a drift here makes every posting undecodable there.
#[test]
fn nzb_posting_cid_wire_shape() {
let cid = compute_nzb_posting_cid(&["[email protected]"]);
assert!(cid.starts_with('b'), "multibase base32-lower prefix");
// Round-trip the base32 body back to bytes to assert the header.
let decoded = base32_lower_decode(&cid[1..]);
assert_eq!(decoded[0], 0x01, "CIDv1");
assert_eq!(&decoded[1..3], &[0x83, 0x20], "varint(0x1003)");
assert_eq!(decoded[3], 0x12, "multihash sha2-256");
assert_eq!(decoded[4], 0x20, "digest length 32");
assert_eq!(decoded.len(), 5 + 32);
// The digest is sha256 over the newline-joined sorted set.
let expect: [u8; 32] = Sha256::digest(b"[email protected]").into();
assert_eq!(&decoded[5..], &expect);
}
}
crates/meta-feeder-sdk/src/lib.rs
+7
View File
@@ -14,6 +14,13 @@
// pure noise for a single-process store, so silence the lint crate-wide.
#![allow(clippy::result_large_err)]
/// pcs-convention default meta-core root a self-publishing feeder targets when
/// neither its dashboard config nor the `META_CORE_URL` env seed sets one. Keeps
/// the modular storage seam out of compose: a standard install just works; a
/// non-standard meta-core is set in the feeder's config UI. Matches the
/// `metacore-app` service the AppStore feeder apps ship on the shared network.
pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000";
pub mod cache;
pub mod common;
pub mod config;
crates/meta-feeder-sdk/src/meta_core.rs
+42
View File
@@ -184,6 +184,48 @@ impl FeederStore {
&self.meta_core_url
}
/// Raw keys containing `contains` (`GET /api/kv/search`). Returns
/// `(keys, truncated)`; `truncated` means the server capped the result and
/// there may be more.
///
/// This is a keyspace scan, so it belongs on rare control-plane paths only —
/// today, invalidating the gateway's search-coverage markers when a feeder's
/// upstream source set changes. Never call it per search.
pub async fn search_keys(&self, contains: &str, limit: usize) -> anyhow::Result<(Vec<String>, bool)> {
let url = format!("{}/api/kv/search", self.meta_core_url);
let resp = self
.http
.get(&url)
.query(&[("contains", contains), ("limit", &limit.to_string())])
.send()
.await?;
let status = resp.status();
if !status.is_success() {
anyhow::bail!("meta-core KV search `{contains}` returned {status}");
}
#[derive(Deserialize)]
struct KvSearchResponse {
#[serde(default)]
keys: Vec<String>,
#[serde(default)]
truncated: bool,
}
let body: KvSearchResponse = resp.json().await?;
Ok((body.keys, body.truncated))
}
/// Delete one raw KV key (`DELETE /api/kv/value?key=`). A missing key is not
/// an error.
pub async fn delete_kv(&self, key: &str) -> anyhow::Result<()> {
let url = format!("{}/api/kv/value", self.meta_core_url);
let resp = self.http.delete(&url).query(&[("key", key)]).send().await?;
let status = resp.status();
if status.is_success() || status == reqwest::StatusCode::NOT_FOUND {
return Ok(());
}
anyhow::bail!("meta-core KV DELETE {key} returned {status}");
}
/// Self-publish a **metadata-only** outcome to meta-core, returning the
/// **sparse** outcome (`record: None`) so the gateway's `(_, None)` branch
/// skips it — no double write. Byte outcomes and already-sparse outcomes are
crates/meta-feeder-sdk/src/plugin.rs
+55
View File
@@ -57,6 +57,30 @@ pub struct HashOutcome {
/// today emits.
/// - `BtV1File` — a single file inside a BitTorrent v1 torrent, custom
/// multicodec `0x1001`. Locator, not content hash: opaque to bitswap.
/// - `NzbRelease` — a Usenet release (a Newznab listing), custom multicodec
/// `nzb-release` `0x1005`. Self-describing locator: the cid embeds `{host,
/// id}` in an identity multihash, opaque to bitswap and redeemed to bytes
/// only by a credentialed meta-share peer, which decodes the locator from the
/// cid and grabs the `.nzb` via `t=get` — no KV side-table.
/// - `CardLocator` — the identity of a *work* (a series, a film) as published by
/// a metadata bridge, custom multicodec `0x1007`. The only family with **no
/// bytes behind it at all**: the record IS the payload. Always paired with
/// `bytes: None`, so it lands in the `(None, Some)` metadata-only branch
/// above. Opaque to bitswap, and deliberately so — there is nothing to seed.
/// - `NzbPosting` — a Usenet posting we **scanned ourselves**, custom multicodec
/// `nzb-posting` `0x1003`. A content-derived digest over the article
/// Message-ID set (not a locator: no host is embedded), so any peer with a
/// plain NNTP provider can fetch it — no `IndexerCred`. Opaque to bitswap: the
/// digest is over the id set, not over any block, so the `.nzb` manifest
/// travels separately as an ordinary sha2-256 cid in the record's `manifest`
/// field. Produced by `hash::compute_nzb_posting_cid`.
/// - `YtVideo` — a **delegated-playback** reference, custom multicodec
/// `yt-video` `0x1008`. Like `CardLocator` it is always paired with
/// `bytes: None` and lands in the `(None, Some)` metadata-only branch, but it
/// means something stronger than "no bytes yet": the bytes **exist and are
/// permanently someone else's**. Nothing is fetched, seeded or
/// content-addressed, and meta-share's byte path REFUSES the cid with a `400`
/// rather than 404ing. Produced by `hash::compute_yt_video_cid`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HashKind {
/// Reserved: no feeder currently emits midhash256 outcomes, but the family
@@ -65,6 +89,10 @@ pub enum HashKind {
Midhash256,
Sha2_256,
BtV1File,
NzbRelease,
CardLocator,
NzbPosting,
YtVideo,
}
/// Startup-only configuration error returned by [`FeederPlugin::configure`].
@@ -101,6 +129,33 @@ pub fn upstream_id_field(upstream_id: &str) -> String {
format!("{upstream_id}id")
}
/// Key-set prefix for the provenance `source/<label>` field (METADATA_KEYS §5).
/// Each member is its own hash field (`source/gateway:nyaa.si = "true"`) so two
/// producers that reach the same content by different routes union without a
/// last-writer-wins clobber. The `<label>` uses `:` internally (not `/`, which
/// is the key-set path separator) — e.g. `gateway:nyaa.si`, `gateway:tribler`.
pub const SOURCE_KEYSET_PREFIX: &str = "source/";
/// Stamp the SDK's default provenance member — `source/gateway:<upstream_id>` —
/// onto a record's fields, but **only if it carries no `source/*` member yet**.
/// This gives every gateway hit a provenance label for free (so meta-watch never
/// falls back to "unknown" for a feeder record), while a plugin that knows a
/// finer origin (torznab → the specific Prowlarr indexer) stamps its own flat
/// `source/*` facets first (`source/gateway`, `source/<protocol>`,
/// `source/<indexer-slug>`) and this default then no-ops.
pub fn stamp_default_source(
fields: &mut std::collections::BTreeMap<String, String>,
upstream_id: &str,
) {
if fields.keys().any(|k| k.starts_with(SOURCE_KEYSET_PREFIX)) {
return;
}
fields.insert(
format!("{SOURCE_KEYSET_PREFIX}gateway:{upstream_id}"),
"true".to_string(),
);
}
/// Static plugin contract. Each enabled `upstream_id` is implemented by
/// exactly one `Box<dyn FeederPlugin>`. Lives behind `&self` in steady state —
/// called concurrently, owns any interior mutability for caches.
crates/meta-feeder-sdk/src/serve.rs
+49 -6
View File
@@ -54,6 +54,10 @@ pub enum HashKindDto {
Midhash256,
Sha2_256,
BtV1File,
NzbRelease,
CardLocator,
NzbPosting,
YtVideo,
}
impl From<HashKind> for HashKindDto {
@@ -62,6 +66,10 @@ impl From<HashKind> for HashKindDto {
HashKind::Midhash256 => HashKindDto::Midhash256,
HashKind::Sha2_256 => HashKindDto::Sha2_256,
HashKind::BtV1File => HashKindDto::BtV1File,
HashKind::NzbRelease => HashKindDto::NzbRelease,
HashKind::CardLocator => HashKindDto::CardLocator,
HashKind::NzbPosting => HashKindDto::NzbPosting,
HashKind::YtVideo => HashKindDto::YtVideo,
}
}
}
@@ -72,6 +80,10 @@ impl From<HashKindDto> for HashKind {
HashKindDto::Midhash256 => HashKind::Midhash256,
HashKindDto::Sha2_256 => HashKind::Sha2_256,
HashKindDto::BtV1File => HashKind::BtV1File,
HashKindDto::NzbRelease => HashKind::NzbRelease,
HashKindDto::CardLocator => HashKind::CardLocator,
HashKindDto::NzbPosting => HashKind::NzbPosting,
HashKindDto::YtVideo => HashKind::YtVideo,
}
}
}
@@ -424,8 +436,20 @@ async fn config_values_put(
std::fs::write(&tmp, body)
.and_then(|_| std::fs::rename(&tmp, &path))
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("write config: {e}")).into_response())?;
info!(target: "meta-feeder", upstream_id = id, path = %path.display(), "feeder config saved (restart to apply)");
Ok(Json(serde_json::json!({ "saved": true, "restart_required": true })))
info!(target: "meta-feeder", upstream_id = id, path = %path.display(), "feeder config saved; scheduling self-restart to apply");
// Apply the new config by restarting the process. We can't hot-swap config
// in-place: plugins hold an exclusive redb cache handle for their whole life,
// so a fresh instance can't reopen the cache while the old one is alive. A
// clean process exit is the reliable apply path — Docker's restart policy
// (`unless-stopped`) brings the feeder back in ~2-3s and `configure()`
// re-reads config.json at startup. Delay briefly so THIS response flushes to
// the client before we exit.
tokio::spawn(async {
tokio::time::sleep(std::time::Duration::from_millis(800)).await;
info!(target: "meta-feeder", "restarting now to apply saved config");
std::process::exit(0);
});
Ok(Json(serde_json::json!({ "saved": true, "reloading": true })))
}
/// `POST /enrich/callback` — sink for the enrichment plugins' completion
@@ -517,6 +541,10 @@ struct DegradedPlugin {
reason: String,
file_types: &'static [&'static str],
content_kinds: &'static [&'static str],
/// Preserved from the wrapped plugin so a degraded/unconfigured feeder still
/// serves its config form — otherwise the dashboard shows "no configuration
/// required" and the operator can never fix the very config that's missing.
schema: crate::config::ConfigSchema,
}
#[async_trait::async_trait]
@@ -557,6 +585,10 @@ impl FeederPlugin for DegradedPlugin {
fn served_content_kinds(&self) -> &'static [&'static str] {
self.content_kinds
}
fn config_schema(&self) -> crate::config::ConfigSchema {
self.schema.clone()
}
}
/// Run each plugin's `configure()` against its per-plugin cache dir and return
@@ -576,6 +608,9 @@ pub fn configure_plugins(
// borrow across `configure(&mut self)` / the move into `out`.
let file_types = plugin.served_file_types();
let content_kinds = plugin.served_content_kinds();
// Preserve the config surface so a degraded stand-in still serves the
// real config form (see DegradedPlugin.schema).
let schema = plugin.config_schema();
let cache_dir = state_dir.join("gateway").join(id);
// Build (don't insert) a degraded stand-in: keeps the container up and
@@ -592,6 +627,7 @@ pub fn configure_plugins(
reason,
file_types,
content_kinds,
schema: schema.clone(),
})
};
@@ -606,12 +642,19 @@ pub fn configure_plugins(
out.insert(id.to_string(), Arc::from(plugin));
}
Err(ConfigError::MissingConfig { plugin: p, what }) => {
// Keep a degraded stand-in (don't drop it) so its config form
// stays reachable from the dashboard and the operator can supply
// the missing config — then restart the feeder to apply it.
warn!(
target: "meta-feeder",
upstream_id = p,
missing = what,
"feeder plugin skipped: required config not supplied; \
other plugins continue"
upstream_id = %p,
missing = %what,
"feeder plugin degraded: required config not supplied; \
config form still served so it can be fixed from the dashboard"
);
out.insert(
id.to_string(),
make_degraded(format!("required config not supplied: {what}")),
);
}
Err(other) => {