feeder rework

2026-08-24 19:08:25 +02:00
parent 2ff7eca1c3
commit 6c71744312
12 changed files with 3565 additions and 1 deletions
@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf
@@ -0,0 +1,59 @@
+name: Publish image
+
+# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
+#   * every push to main  -> the moving `main` tag
+#   * every `v*` tag       -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
+# The build context is the repo root (the workspace) so the vendored
+# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
+# feeder-plugin/paper-feeder/.
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,format=short
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./feeder-plugin/internetarchive-feeder/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
@@ -0,0 +1,9 @@
+[workspace]
+resolver = "2"
+members = ["feeder-plugin/*"]
+
+[workspace.package]
+edition = "2021"
+rust-version = "1.85"
+license = "MIT"
+repository = "https://github.com/worph/meta-feeder-internetarchive"
@@ -1 +1,17 @@
-# meta-feeder-internetarchive
+# meta-feeder-internetarchive
+
+Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless.
+
+**Upstream:** Internet Archive (archive.org)
+**Role:** bytes
+
+A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin`
+from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk).
+
+```bash
+cargo build --release --bin internetarchive-feeder
+cargo test
+```
+
+Scope and naming conventions: `docs/project-architecture/feeder-architecture.md`
+in the meta-root.
@@ -0,0 +1,42 @@
+[package]
+name = "internetarchive-feeder"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish = false
+
+# Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless.
+# Upstream: Internet Archive (archive.org)
+
+[lib]
+name = "internetarchive_feeder"
+path = "src/lib.rs"
+
+[[bin]]
+name = "internetarchive-feeder"
+path = "src/main.rs"
+
+[dependencies]
+# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to:
+#   meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "<sha>" }
+meta-feeder-sdk = { path = "../../../meta-feeder-sdk" }
+
+tokio = { version = "1.40", features = ["full"] }
+axum = { version = "0.7", features = ["json"] }
+anyhow = "1"
+async-trait = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
+bytes = "1"
+futures = "0.3"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+redb = "2"
+base64 = "0.22"
+
+[dev-dependencies]
+wiremock = "0.6"
+tempfile = "3"
+tokio = { version = "1.40", features = ["full", "test-util"] }
@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+# internetarchive-feeder sidecar. Multi-stage build; build context is this repo's root.
+# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied
+# in — `git` must be present in the builder for cargo to fetch it.
+
+FROM rust:1.89-slim-bookworm AS builder
+WORKDIR /build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        pkg-config libssl-dev ca-certificates git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY Cargo.toml rust-toolchain.toml ./
+COPY feeder-plugin/ feeder-plugin/
+
+RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
+    --mount=type=cache,id=internetarchive-feeder-target,target=/build/target \
+    cargo build --release --bin internetarchive-feeder \
+    && cp target/release/internetarchive-feeder /internetarchive-feeder
+
+# -----------------------------------------------------------------------------
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /internetarchive-feeder /usr/local/bin/internetarchive-feeder
+
+RUN useradd --system --uid 10001 --no-create-home metafeeder \
+    && mkdir -p /data/meta-feeder \
+    && chown metafeeder:metafeeder /data/meta-feeder
+
+USER metafeeder
+WORKDIR /data/meta-feeder
+
+EXPOSE 8080
+
+ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
+    META_FEEDER_STATE_DIR=/data/meta-feeder \
+    RUST_LOG=info
+
+ENTRYPOINT ["/usr/local/bin/internetarchive-feeder"]
@@ -0,0 +1,87 @@
+//! Constants for `internetarchive-feeder`.
+
+/// User-Agent for outbound HTTP.
+///
+/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent
+/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to
+/// misdiagnose. Identify the application and carry contact information.
+pub const USER_AGENT: &str = concat!(
+    "internetarchive-feeder/",
+    env!("CARGO_PKG_VERSION"),
+    " ( https://github.com/worph/meta-feeder-internetarchive )"
+);
+
+/// Internet Archive advanced-search endpoint. Keyless.
+pub const IA_SEARCH_URL: &str = "https://archive.org/advancedsearch.php";
+/// Internet Archive per-item file-listing endpoint (`/metadata/<identifier>`).
+pub const IA_METADATA_BASE: &str = "https://archive.org/metadata";
+/// Internet Archive file-download base (`/download/<identifier>/<file>`).
+pub const IA_DOWNLOAD_BASE: &str = "https://archive.org/download";
+/// The archive's own thumbnail service, keyed by item identifier.
+///
+/// ⚠ **Without this the entire free tier is invisible.** The consumer's quality
+/// gate requires a cover, and an archive item's artwork is an ordinary file in
+/// its file list under an unpredictable name — so a first cut emitted no
+/// artwork field at all and every free-tier track landed in the raw bucket. The
+/// cold-start row, whose whole purpose is to be full on a client with nothing
+/// configured, was permanently empty.
+///
+/// This endpoint always answers for a real item (verified: 200 image/png), so
+/// it is the one predictable cover source the archive has.
+pub const IA_THUMB_BASE: &str = "https://archive.org/services/img";
+/// Sustained Internet Archive request rate. They publish no hard number and
+/// ask for reasonable behaviour; this is deliberately modest because a single
+/// item listing can be large.
+pub const IA_RATE_PER_SEC: f64 = 3.0;
+/// Internet Archive burst ceiling.
+pub const IA_BURST: f64 = 6.0;
+/// Request timeout for an Internet Archive metadata/search call (seconds).
+pub const IA_TIMEOUT_SECS: u64 = 30;
+/// Timeout for downloading one audio file's bytes (seconds). Generous: a
+/// lossless live-set track can be 100 MB.
+pub const IA_FETCH_TIMEOUT_SECS: u64 = 600;
+/// Hard ceiling on the bytes this feeder will pull for a single track.
+///
+/// The gateway core holds the whole outcome in memory before seeding it, so an
+/// unbounded fetch is an OOM waiting to happen on a 128 MB feeder. A 200 MB
+/// ceiling clears any normal lossless track (a 24/96 FLAC of a 10-minute piece
+/// is ~200 MB at the extreme) while refusing a whole-concert single file.
+pub const IA_MAX_TRACK_BYTES: u64 = 200 * 1024 * 1024;
+/// The Internet Archive collections this feeder searches.
+///
+/// Deliberately an allow-list rather than "all of `mediatype:audio`". The
+/// archive holds a great deal of audio whose redistribution status is unclear
+/// (uploads of commercial recordings, radio airchecks). These four are the
+/// collections whose licensing is either explicit per item or structurally
+/// clean, which is what makes the `licence` field on the emitted record
+/// meaningful rather than decorative:
+///
+/// - `etree` — the Live Music Archive: taper recordings of bands who have
+///   explicitly permitted taping and distribution.
+/// - `netlabels` — net-label releases, uniformly Creative Commons.
+/// - `audio_music` — the general music collection; per-item `licenseurl` is
+///   what gates an individual record (see `internetarchive.rs`).
+/// - `78rpm` — digitised 78s, overwhelmingly public domain by age.
+pub const IA_COLLECTIONS: &[&str] = &["etree", "netlabels", "audio_music", "78rpm"];
+/// Archive `mediatype` values that hold music.
+///
+/// ⚠ **`etree` is a mediatype, not only a collection**, and missing that is the
+/// easiest way to silently lose most of the free tier. Live Music Archive items
+/// are `mediatype:etree`; a query restricted to `mediatype:(audio)` matches
+/// **zero** of them. Measured on the live endpoint:
+///
+/// | query | numFound |
+/// |---|---|
+/// | `mediatype:(audio) AND collection:(etree)` | **0** |
+/// | `mediatype:(etree)` | 293,738 |
+/// | `collection:(etree) AND creator:("Grateful Dead")` | 18,334 |
+///
+/// The failure is quiet — an `audio`-only query still returns a handful of
+/// `audio_music` items, so the search looks like it works and merely appears to
+/// have a thin catalogue.
+pub const IA_MEDIATYPES: &[&str] = &["audio", "etree"];
+/// Audio file extensions the Internet Archive tier will emit as tracks,
+/// preferred order (best quality first). The archive stores several
+/// derivatives of the same recording; emitting all of them would publish four
+/// records for one track.
+pub const IA_AUDIO_EXTS: &[&str] = &["flac", "ogg", "opus", "m4a", "mp3"];
@@ -0,0 +1,6 @@
+//! `internetarchive-feeder` library surface.
+//!
+//! Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless.
+
+pub mod consts;
+pub mod internetarchive;
@@ -0,0 +1,33 @@
+//! `internetarchive-feeder` — a single-upstream feeder sidecar.
+//!
+//! Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless.
+//!
+//! Upstream: Internet Archive (archive.org)
+//!
+//! Env:
+//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
+//! - `META_FEEDER_STATE_DIR`   — per-plugin cache root (default `/data/meta-feeder`)
+//! - `RUST_LOG`                — tracing filter (default `info`)
+
+use std::net::SocketAddr;
+
+use meta_feeder_sdk::plugin::FeederPlugin;
+use meta_feeder_sdk::serve_feeders;
+use internetarchive_feeder::internetarchive::InternetArchivePlugin;
+use tracing_subscriber::EnvFilter;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+        .init();
+
+    let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
+        .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
+        .parse()?;
+    let state_dir =
+        std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
+
+    let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(InternetArchivePlugin::new())];
+    serve_feeders(plugins, state_dir, listen).await
+}
@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.89"
+components = ["rustfmt", "clippy"]