feeder rework

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

.github/workflows/publish.yml

@@ -0,0 +1,59 @@
+name: Publish image
+
+# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
+#   * every push to main  -> the moving `main` tag
+#   * every `v*` tag       -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
+# The build context is the repo root (the workspace) so the vendored
+# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
+# feeder-plugin/paper-feeder/.
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,format=short
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./feeder-plugin/jamendo-feeder/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Cargo.toml

@@ -0,0 +1,9 @@
+[workspace]
+resolver = "2"
+members = ["feeder-plugin/*"]
+
+[workspace.package]
+edition = "2021"
+rust-version = "1.85"
+license = "MIT"
+repository = "https://github.com/worph/meta-feeder-jamendo"

README.md

@@ -1 +1,17 @@
 # meta-feeder-jamendo
+
+Jamendo Creative-Commons music. Opt-in: needs a client id.
+
+**Upstream:** Jamendo API v3.0 (api.jamendo.com)
+**Role:** bytes
+
+A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin`
+from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk).
+
+```bash
+cargo build --release --bin jamendo-feeder
+cargo test
+```
+
+Scope and naming conventions: `docs/project-architecture/feeder-architecture.md`
+in the meta-root.

feeder-plugin/jamendo-feeder/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "jamendo-feeder"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish = false
+
+# Jamendo Creative-Commons music. Opt-in: needs a client id.
+# Upstream: Jamendo API v3.0 (api.jamendo.com)
+
+[lib]
+name = "jamendo_feeder"
+path = "src/lib.rs"
+
+[[bin]]
+name = "jamendo-feeder"
+path = "src/main.rs"
+
+[dependencies]
+# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to:
+#   meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "<sha>" }
+meta-feeder-sdk = { path = "../../../meta-feeder-sdk" }
+
+tokio = { version = "1.40", features = ["full"] }
+axum = { version = "0.7", features = ["json"] }
+anyhow = "1"
+async-trait = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
+bytes = "1"
+futures = "0.3"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+redb = "2"
+base64 = "0.22"
+
+[dev-dependencies]
+wiremock = "0.6"
+tempfile = "3"
+tokio = { version = "1.40", features = ["full", "test-util"] }

feeder-plugin/jamendo-feeder/Dockerfile

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+# jamendo-feeder sidecar. Multi-stage build; build context is this repo's root.
+# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied
+# in — `git` must be present in the builder for cargo to fetch it.
+
+FROM rust:1.89-slim-bookworm AS builder
+WORKDIR /build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        pkg-config libssl-dev ca-certificates git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY Cargo.toml rust-toolchain.toml ./
+COPY feeder-plugin/ feeder-plugin/
+
+RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
+    --mount=type=cache,id=jamendo-feeder-target,target=/build/target \
+    cargo build --release --bin jamendo-feeder \
+    && cp target/release/jamendo-feeder /jamendo-feeder
+
+# -----------------------------------------------------------------------------
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /jamendo-feeder /usr/local/bin/jamendo-feeder
+
+RUN useradd --system --uid 10001 --no-create-home metafeeder \
+    && mkdir -p /data/meta-feeder \
+    && chown metafeeder:metafeeder /data/meta-feeder
+
+USER metafeeder
+WORKDIR /data/meta-feeder
+
+EXPOSE 8080
+
+ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
+    META_FEEDER_STATE_DIR=/data/meta-feeder \
+    RUST_LOG=info
+
+ENTRYPOINT ["/usr/local/bin/jamendo-feeder"]

feeder-plugin/jamendo-feeder/src/consts.rs

@@ -0,0 +1,104 @@
+//! Constants for `jamendo-feeder`.
+
+/// User-Agent for outbound HTTP.
+///
+/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent
+/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to
+/// misdiagnose. Identify the application and carry contact information.
+pub const USER_AGENT: &str = concat!(
+    "jamendo-feeder/",
+    env!("CARGO_PKG_VERSION"),
+    " ( https://github.com/worph/meta-feeder-jamendo )"
+);
+
+/// Jamendo web API v3 base.
+pub const JAMENDO_API_BASE: &str = "https://api.jamendo.com/v3.0";
+/// Streaming/download format segment asked of the API.
+///
+/// `mp32` is Jamendo's high-bitrate MP3 derivative. Deliberately not `flac`:
+/// this tier's bytes are fetched, buffered whole in the gateway core, and then
+/// seeded to every peer that asks — an order-of-magnitude size increase for a
+/// quality difference no one has asked for.
+pub const JAMENDO_AUDIO_FORMAT: &str = "mp32";
+/// Sustained Jamendo request rate.
+///
+/// Jamendo publishes no per-second ceiling — only the monthly quota below — so
+/// this is a courtesy limit, not a documented one. It is deliberately modest:
+/// the binding constraint here is the *month*, and a burst that fills a page
+/// faster buys nothing but a nearer exhaustion date.
+pub const JAMENDO_RATE_PER_SEC: f64 = 2.0;
+/// Jamendo burst ceiling.
+pub const JAMENDO_BURST: f64 = 4.0;
+/// ⚠ **The published non-commercial quota: 35,000 API requests per MONTH.**
+///
+/// This is the one number that makes Jamendo unlike every other upstream in
+/// this crate, and getting it wrong is not recoverable by waiting a second.
+/// MusicBrainz's 1/s is a *rate*: exceed it and you are throttled, back off and
+/// you continue. This is a *budget*: spend it and the upstream is gone until
+/// the calendar turns.
+///
+/// A token bucket cannot express that — it refills. So this tier carries a
+/// **persisted monthly counter** in the plugin cache (`MusicCache::
+/// jamendo_quota_spend`) alongside the per-second bucket, and refuses calls
+/// rather than discovering the ceiling from a 429 three weeks in.
+pub const JAMENDO_MONTHLY_QUOTA: u64 = 35_000;
+/// Default share of [`JAMENDO_MONTHLY_QUOTA`] this feeder will spend.
+///
+/// Headroom is deliberate and serves two distinct purposes:
+///
+/// 1. The counter is **per feeder instance**, but the quota is per *client_id*.
+///    Two gateways configured with the same credential each believe they own
+///    the whole month. Until the counter is shared (it would have to live in
+///    meta-core to be), the margin is what keeps a two-peer deployment from
+///    silently overspending.
+/// 2. It leaves a reserve for `compute_outcomes` — the calls a user is
+///    actually waiting on — after a month of browse rows.
+pub const JAMENDO_DEFAULT_MONTHLY_BUDGET: u64 = 30_000;
+/// Request timeout for a Jamendo API call (seconds).
+pub const JAMENDO_TIMEOUT_SECS: u64 = 20;
+/// Budget-permit deadline for a query somebody is waiting on.
+pub const JAMENDO_WAIT_DEADLINE_SECS: u64 = 20;
+/// Timeout for downloading one track's bytes (seconds).
+pub const JAMENDO_FETCH_TIMEOUT_SECS: u64 = 300;
+/// Hard ceiling on the bytes this tier will pull for a single track.
+///
+/// Lower than the Internet Archive's 200 MB because this tier fetches MP3
+/// only, never a lossless master. A 100 MB MP3 is roughly a three-hour
+/// recording — past any plausible track and into "someone published a DJ set
+/// as one file" territory, which the gateway core would buffer whole.
+pub const JAMENDO_MAX_TRACK_BYTES: u64 = 100 * 1024 * 1024;
+/// Max `limit` the API accepts on one page.
+pub const JAMENDO_PAGE_MAX: usize = 200;
+/// Ordering used for the keyword-less browse row (the free-tier home row).
+///
+/// The consumer's cold-start row carries no text, exactly like the Internet
+/// Archive tier's. Jamendo answers an order-only query with its catalogue in
+/// popularity order, which is what that row wants.
+pub const JAMENDO_BROWSE_ORDER: &str = "popularity_total_desc";
+/// Extra attempts made when Jamendo answers a query with an empty page.
+///
+/// ⚠ **Not defensive padding — a measured mitigation for an upstream that
+/// intermittently lies.** Jamendo answers a perfectly good query with
+/// `HTTP 200`, `headers.status: "success"` and `results_count: 0` a large
+/// fraction of the time. Measured 2026-08-21 by firing one unchanged URL
+/// repeatedly:
+///
+/// | query shape | empty responses |
+/// |---|---|
+/// | `search=love` (×12) | 2 |
+/// | `namesearch=love` (×8) | 3 |
+/// | `order=popularity_total_desc` (×8) | **4** |
+///
+/// It looks like one replica behind their load balancer serving an empty
+/// index. Nothing in the envelope distinguishes it from a genuine no-match, so
+/// a single-attempt client turns a third of every Jamendo-backed row into a
+/// blank — and the consumer's sticky per-row merge would quietly paper over it
+/// as "the row is sometimes short".
+///
+/// Two retries take the odds of a false empty from ~1-in-3 to ~1-in-27. The
+/// quota cost is bounded because a retry only happens on an empty page:
+/// roughly 0.5 extra calls per query at the measured rate.
+pub const JAMENDO_EMPTY_RETRIES: u32 = 2;
+/// Pause between the attempts above. Long enough to land on a different
+/// backend, short enough that three attempts stay inside a search timeout.
+pub const JAMENDO_EMPTY_RETRY_DELAY_MS: u64 = 250;

feeder-plugin/jamendo-feeder/src/lib.rs

@@ -0,0 +1,7 @@
+//! `jamendo-feeder` library surface.
+//!
+//! Jamendo Creative-Commons music. Opt-in: needs a client id.
+
+pub mod consts;
+pub mod jamendo;
+pub mod shared;

feeder-plugin/jamendo-feeder/src/main.rs

@@ -0,0 +1,33 @@
+//! `jamendo-feeder` — a single-upstream feeder sidecar.
+//!
+//! Jamendo Creative-Commons music. Opt-in: needs a client id.
+//!
+//! Upstream: Jamendo API v3.0 (api.jamendo.com)
+//!
+//! Env:
+//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
+//! - `META_FEEDER_STATE_DIR`   — per-plugin cache root (default `/data/meta-feeder`)
+//! - `RUST_LOG`                — tracing filter (default `info`)
+
+use std::net::SocketAddr;
+
+use meta_feeder_sdk::plugin::FeederPlugin;
+use meta_feeder_sdk::serve_feeders;
+use jamendo_feeder::jamendo::JamendoPlugin;
+use tracing_subscriber::EnvFilter;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+        .init();
+
+    let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
+        .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
+        .parse()?;
+    let state_dir =
+        std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
+
+    let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(JamendoPlugin::new())];
+    serve_feeders(plugins, state_dir, listen).await
+}

feeder-plugin/jamendo-feeder/src/shared.rs

@@ -0,0 +1,150 @@
+//! Helpers shared by every **content-tier** upstream in this crate.
+//!
+//! `internetarchive` and `jamendo` are independent bridges to unrelated
+//! services, but they answer the same shape of question — "which playable
+//! files match this query?" — and so they need the same three pieces of
+//! plumbing. These began as private items in `internetarchive.rs`; they are
+//! here rather than duplicated so that a fix to the filter-echo rule (a subtle
+//! one — see below) cannot land in one upstream and be forgotten in the other.
+//!
+//! ⚠ `internetarchive.rs` still carries its own private copies of
+//! `echo_filters` / `wants_content_kind` and its own inline identity-filter
+//! list. Collapsing them onto this module is a mechanical follow-up, deferred
+//! only because that file had unrelated edits in flight when this landed.
+//!
+//! The card tier deliberately does NOT use these: a card is a *work*, not a
+//! file, and its projection has its own filter rules in `card.rs`.
+
+use std::collections::BTreeMap;
+
+use meta_feeder_sdk::plugin::GatewayQuery;
+
+/// Echo back structured filters the record did not already set, so the record
+/// survives `record_matches` at the gateway and again at meta-search.
+///
+/// The two exclusions are not arbitrary:
+///
+/// - **`languages`** and **`genres`** are `key-set` fields
+///   (`genres/<Name> = "true"`), and `record_matches` has a dedicated arm for
+///   each. Echoing a flat `genres = "jazz,pop"` value would write the legacy
+///   csv-set shape that `METADATA_KEYS.md` §14.12 forbids for new writers, and
+///   it would not help the record match anyway.
+/// - A key the record **already set** is never overwritten. The record's own
+///   value is the truth; the filter is only what was asked for.
+pub(crate) fn echo_filters(
+    fields: &mut BTreeMap<String, String>,
+    query_filters: &BTreeMap<String, Vec<String>>,
+) {
+    for (key, allowed) in query_filters {
+        if key == "languages" || key == "genres" || fields.contains_key(key) || allowed.is_empty() {
+            continue;
+        }
+        fields.insert(key.clone(), allowed.join(","));
+    }
+}
+
+/// Does the query ask for this content kind? No filter means everything the
+/// calling plugin serves.
+pub(crate) fn wants_content_kind(query: &GatewayQuery, kind: &str) -> bool {
+    match query.filters.get("contentKind") {
+        None => true,
+        Some(values) if values.is_empty() => true,
+        Some(values) => values.iter().any(|v| v.trim().eq_ignore_ascii_case(kind)),
+    }
+}
+
+/// Filters that name a *specific work* by id.
+///
+/// Neither content upstream holds MusicBrainz identifiers, so a query carrying
+/// one is a question they cannot answer.
+const IDENTITY_FILTERS: &[&str] = &[
+    "mbReleaseGroupId",
+    "mbReleaseId",
+    "mbArtistId",
+    "mbRecordingId",
+    "isrc",
+    "tmdbid",
+    "imdbid",
+];
+
+/// ⚠ **An identity-anchored query a content upstream cannot satisfy must
+/// return NOTHING**, not a free-text browse.
+///
+/// A query like `mbReleaseGroupId:<uuid> fileType:audio` carries no free text
+/// for a keyword upstream to search on. Falling through to the keyword path
+/// then means an *empty* search — which both upstreams happily answer with
+/// their most-popular items.
+///
+/// Observed live on the Internet Archive tier before this guard existed:
+/// opening a BTS album filled its page with sixty Grateful Dead and
+/// shortwave-numbers-station recordings, every one of them a confident-looking
+/// answer to a question nobody asked. A consumer's album page is exactly this
+/// query shape, so the failure is not an edge case — it is the normal
+/// detail-page request.
+pub(crate) fn is_identity_anchored(query: &GatewayQuery) -> bool {
+    IDENTITY_FILTERS
+        .iter()
+        .any(|k| query.filters.contains_key(*k))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn query(filters: &[(&str, &[&str])]) -> GatewayQuery {
+        let mut q = GatewayQuery::from_free_text("");
+        for (k, vs) in filters {
+            q.filters.insert(
+                (*k).to_string(),
+                vs.iter().map(|v| (*v).to_string()).collect(),
+            );
+        }
+        q
+    }
+
+    #[test]
+    fn identity_anchored_queries_are_recognised() {
+        assert!(is_identity_anchored(&query(&[(
+            "mbReleaseGroupId",
+            &["f5093c06"]
+        )])));
+        assert!(is_identity_anchored(&query(&[("isrc", &["USRC17607839"])])));
+        // A plain typed query is not anchored — it is the keyword path.
+        assert!(!is_identity_anchored(&query(&[
+            ("fileType", &["audio"]),
+            ("contentKind", &["track"])
+        ])));
+    }
+
+    #[test]
+    fn content_kind_absent_means_everything() {
+        let q = query(&[("fileType", &["audio"])]);
+        assert!(wants_content_kind(&q, "track"));
+        assert!(wants_content_kind(&q, "pack"));
+
+        let q = query(&[("contentKind", &["pack"])]);
+        assert!(!wants_content_kind(&q, "track"));
+        assert!(wants_content_kind(&q, "pack"));
+    }
+
+    /// The key-set exclusions. Echoing a flat `genres` value would write the
+    /// forbidden csv-set shape and would not make the record match anyway.
+    #[test]
+    fn key_set_filters_are_never_echoed_flat() {
+        let mut fields = BTreeMap::new();
+        fields.insert("fileType".to_string(), "audio".to_string());
+        let mut filters = BTreeMap::new();
+        filters.insert("genres".to_string(), vec!["jazz".to_string()]);
+        filters.insert("languages".to_string(), vec!["eng".to_string()]);
+        filters.insert("quality".to_string(), vec!["lossless".to_string()]);
+        // `fileType` is already set by the record and must not be clobbered.
+        filters.insert("fileType".to_string(), vec!["video".to_string()]);
+
+        echo_filters(&mut fields, &filters);
+
+        assert!(!fields.contains_key("genres"));
+        assert!(!fields.contains_key("languages"));
+        assert_eq!(fields.get("quality").map(String::as_str), Some("lossless"));
+        assert_eq!(fields.get("fileType").map(String::as_str), Some("audio"));
+    }
+}

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.89"
+components = ["rustfmt", "clippy"]