//! The `FeederPlugin` trait + supporting types. //! //! A feeder plugin bridges one external upstream (`gutenberg`, `arxiv`, //! `torznab`, …) into the MetaMesh network. It **finds** records and //! **fetches** bytes; it does NOT hash-into-a-blockstore, store back into //! meta-core, or speak libp2p — those stay in the gateway core. This is why //! `FeederPlugin` has no `set_blockstore` / `set_tmdb_budget` hooks (the //! gateway's `GatewayPlugin` carried them): preview-seeding moves to the core, //! and a feeder that needs a TMDB budget owns it internally. //! //! The `serve_feeders` harness ([`crate::serve`]) wraps any set of //! `FeederPlugin`s in the feeder HTTP contract; the gateway core's //! `RemoteFeederPlugin` is the matching HTTP client. use std::collections::HashMap; use std::path::Path; use async_trait::async_trait; use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; // Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the // gateway's original `plugin.rs` re-exported it the same way). pub use crate::query::GatewayQuery; use crate::query::GatewaySearchEvent; use crate::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; /// One element of what [`FeederPlugin::compute_outcomes`] returns. /// /// The `hash` is the value shipped to the gateway core. The other fields are /// signals the core's auto-store uses to persist the content into meta-core. /// The core's three-branch auto-store routes on `(bytes, record)`: /// - `(Some, Some)` — full store: WebDAV PUT + metadata PUT with `filePath`. /// - `(None, Some)` — metadata-only PUT (no local bytes; e.g. torznab samples /// the middle 1 MiB of a multi-GiB video and surfaces only the record). /// - `(_, None)` — skip. Cache hits land here. /// /// `file_extension` MUST be `None` when `bytes` is `None`. #[derive(Debug)] pub struct HashOutcome { pub hash: Hash, /// Which hash family this CID belongs to. Drives the core's routing — /// the bitswap blockstore seed only fires for `Sha2_256`. pub hash_kind: HashKind, pub bytes: Option, pub record: Option, /// File extension to append when naming the WebDAV-side blob (e.g. /// `"epub"`). No leading dot. MUST be `None` when `bytes` is `None`. pub file_extension: Option, } /// Hash family discriminator on [`HashOutcome`]. /// /// - `Midhash256` — size-prefix-plus-middle-1MB-sample, custom multicodec /// `0x1000`. Fast, MetaMesh-internal, not retrievable via public IPFS. /// - `Sha2_256` — standard IPFS CIDv1 over full bytes. Retrievable via bitswap /// once the bytes are in the core's blockstore. What every shipping plugin /// today emits. /// - `BtV1File` — a single file inside a BitTorrent v1 torrent, custom /// multicodec `0x1001`. Locator, not content hash: opaque to bitswap. /// - `NzbRelease` — a Usenet release (a Newznab listing), custom multicodec /// `nzb-release` `0x1005`. Self-describing locator: the cid embeds `{host, /// id}` in an identity multihash, opaque to bitswap and redeemed to bytes /// only by a credentialed meta-share peer, which decodes the locator from the /// cid and grabs the `.nzb` via `t=get` — no KV side-table. /// - `CardLocator` — the identity of a *work* (a series, a film) as published by /// a metadata bridge, custom multicodec `0x1007`. The only family with **no /// bytes behind it at all**: the record IS the payload. Always paired with /// `bytes: None`, so it lands in the `(None, Some)` metadata-only branch /// above. Opaque to bitswap, and deliberately so — there is nothing to seed. /// - `NzbPosting` — a Usenet posting we **scanned ourselves**, custom multicodec /// `nzb-posting` `0x1003`. A content-derived digest over the article /// Message-ID set (not a locator: no host is embedded), so any peer with a /// plain NNTP provider can fetch it — no `IndexerCred`. Opaque to bitswap: the /// digest is over the id set, not over any block, so the `.nzb` manifest /// travels separately as an ordinary sha2-256 cid in the record's `manifest` /// field. Produced by `hash::compute_nzb_posting_cid`. /// - `YtVideo` — a **delegated-playback** reference, custom multicodec /// `yt-video` `0x1008`. Like `CardLocator` it is always paired with /// `bytes: None` and lands in the `(None, Some)` metadata-only branch, but it /// means something stronger than "no bytes yet": the bytes **exist and are /// permanently someone else's**. Nothing is fetched, seeded or /// content-addressed, and meta-share's byte path REFUSES the cid with a `400` /// rather than 404ing. Produced by `hash::compute_yt_video_cid`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum HashKind { /// Reserved: no feeder currently emits midhash256 outcomes, but the family /// is real across meta-core / meta-share, so the variant stays. #[allow(dead_code)] Midhash256, Sha2_256, BtV1File, NzbRelease, CardLocator, NzbPosting, YtVideo, } /// `RedeemClaim::codec` for a Newznab release locator (`0x1005`). pub const REDEEM_CODEC_NZB_RELEASE: &str = "nzb-release"; /// `RedeemClaim::codec` for a `provider-file` locator (`0x100A`). pub const REDEEM_CODEC_PROVIDER_FILE: &str = "provider-file"; /// `RedeemClaim::field` naming the `.nzb` manifest pointer on a release record. pub const POINTER_FIELD_MANIFEST: &str = "manifest"; /// `RedeemClaim::field` naming the fetched-file pointer on a locator record. pub const POINTER_FIELD_FILE: &str = "file"; /// A plugin's claim that it can **redeem** a locator family: turn a locator cid /// into the bytes it names, through `POST /compute` with `record_id` = the /// locator cid. /// /// Redeeming usually spends a metered upstream quota (a Newznab `t=get` grab, /// an OpenSubtitles `/download`), so the gateway routes a redeem only to the /// plugins claiming the cid's codec, one at a time, and only on a real play. It /// stores the returned bytes and writes their cid onto the locator record under /// [`field`](Self::field), so the quota is spent once per file. /// /// Served at `GET /manifest` (per plugin) and `GET /redeems`. Reflects the /// plugin's *current* config: a plugin with no credential for a family claims /// nothing for it. #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] pub struct RedeemClaim { /// Locator family: [`REDEEM_CODEC_NZB_RELEASE`] or [`REDEEM_CODEC_PROVIDER_FILE`]. pub codec: String, /// Pointer field the gateway writes the redeemed bytes' cid under: /// [`POINTER_FIELD_MANIFEST`] or [`POINTER_FIELD_FILE`]. pub field: String, /// Indexer hosts this plugin holds a key for (`nzb-release`) — bare /// authority, lowercase, never the key itself. #[serde(default)] pub hosts: Vec, /// Provider sources this plugin redeems (`provider-file`), e.g. `opensubtitles`. #[serde(default)] pub sources: Vec, } impl RedeemClaim { /// Claim `nzb-release` locators for these indexer hosts. pub fn nzb_release(hosts: Vec) -> Self { Self { codec: REDEEM_CODEC_NZB_RELEASE.to_string(), field: POINTER_FIELD_MANIFEST.to_string(), hosts, sources: Vec::new(), } } /// Claim `provider-file` locators for these sources. pub fn provider_file(sources: Vec) -> Self { Self { codec: REDEEM_CODEC_PROVIDER_FILE.to_string(), field: POINTER_FIELD_FILE.to_string(), hosts: Vec::new(), sources, } } } /// Startup-only configuration error returned by [`FeederPlugin::configure`]. #[derive(Debug, thiserror::Error)] pub enum ConfigError { #[error("plugin {plugin}: required config missing: {what}")] MissingConfig { plugin: &'static str, /// Human-readable description of the missing setting and where to /// supply it. Used by the harness's soft-skip warning. what: &'static str, }, #[error("plugin {plugin}: cache dir setup failed: {source}")] CacheSetup { plugin: &'static str, #[source] source: std::io::Error, }, #[error("plugin {plugin}: {source}")] Other { plugin: &'static str, #[source] source: anyhow::Error, }, } /// Canonical field name for an upstream's stable per-record identifier. /// Every plugin's `DiscoveryRecord.fields` MUST include this key /// (`gutenbergid:11`, `tmdbid:31910`, `arxivid:2106.07447`). The rule is /// `format!("{upstream_id}id")` — lowercase, no separator. pub fn upstream_id_field(upstream_id: &str) -> String { format!("{upstream_id}id") } /// Key-set prefix for the provenance `source/