feeder rework

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

.github/workflows/publish.yml

@@ -0,0 +1,59 @@
+name: Publish image
+
+# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
+#   * every push to main  -> the moving `main` tag
+#   * every `v*` tag       -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
+# The build context is the repo root (the workspace) so the vendored
+# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
+# feeder-plugin/paper-feeder/.
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,format=short
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./feeder-plugin/youtube-feeder/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Cargo.toml

@@ -0,0 +1,9 @@
+[workspace]
+resolver = "2"
+members = ["feeder-plugin/*"]
+
+[workspace.package]
+edition = "2021"
+rust-version = "1.85"
+license = "MIT"
+repository = "https://github.com/worph/meta-feeder-youtube"

README.md

@@ -1 +1,17 @@
-# meta-feeder-youtube
+# meta-feeder-youtube
+
+YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded.
+
+**Upstream:** YouTube via Piped instances
+**Role:** delegated
+
+A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin`
+from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk).
+
+```bash
+cargo build --release --bin youtube-feeder
+cargo test
+```
+
+Scope and naming conventions: `docs/project-architecture/feeder-architecture.md`
+in the meta-root.

feeder-plugin/youtube-feeder/Cargo.toml

@@ -0,0 +1,45 @@
+[package]
+name = "youtube-feeder"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish = false
+
+# YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded.
+# Upstream: YouTube via Piped instances
+
+[lib]
+name = "youtube_feeder"
+path = "src/lib.rs"
+
+[[bin]]
+name = "youtube-feeder"
+path = "src/main.rs"
+
+[dependencies]
+# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to:
+#   meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "<sha>" }
+meta-feeder-sdk = { path = "../../../meta-feeder-sdk" }
+
+tokio = { version = "1.40", features = ["full"] }
+axum = { version = "0.7", features = ["json"] }
+anyhow = "1"
+async-trait = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
+bytes = "1"
+futures = "0.3"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+redb = "2"
+base64 = "0.22"
+# The MusicBrainz client, from the crate that owns it. Only the *music*
+# path uses it; the video path never touches MusicBrainz.
+musicbrainz-feeder = { path = "../../../meta-feeder-musicbrainz/feeder-plugin/musicbrainz-feeder" }
+
+[dev-dependencies]
+wiremock = "0.6"
+tempfile = "3"
+tokio = { version = "1.40", features = ["full", "test-util"] }

feeder-plugin/youtube-feeder/Dockerfile

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+# youtube-feeder sidecar. Multi-stage build; build context is this repo's root.
+# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied
+# in — `git` must be present in the builder for cargo to fetch it.
+
+FROM rust:1.89-slim-bookworm AS builder
+WORKDIR /build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        pkg-config libssl-dev ca-certificates git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY Cargo.toml rust-toolchain.toml ./
+COPY feeder-plugin/ feeder-plugin/
+
+RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
+    --mount=type=cache,id=youtube-feeder-target,target=/build/target \
+    cargo build --release --bin youtube-feeder \
+    && cp target/release/youtube-feeder /youtube-feeder
+
+# -----------------------------------------------------------------------------
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /youtube-feeder /usr/local/bin/youtube-feeder
+
+RUN useradd --system --uid 10001 --no-create-home metafeeder \
+    && mkdir -p /data/meta-feeder \
+    && chown metafeeder:metafeeder /data/meta-feeder
+
+USER metafeeder
+WORKDIR /data/meta-feeder
+
+EXPOSE 8080
+
+ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
+    META_FEEDER_STATE_DIR=/data/meta-feeder \
+    RUST_LOG=info
+
+ENTRYPOINT ["/usr/local/bin/youtube-feeder"]

feeder-plugin/youtube-feeder/src/consts.rs

@@ -0,0 +1,57 @@
+//! Constants for `youtube-feeder`.
+
+/// User-Agent for outbound HTTP.
+///
+/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent
+/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to
+/// misdiagnose. Identify the application and carry contact information.
+pub const USER_AGENT: &str = concat!(
+    "youtube-feeder/",
+    env!("CARGO_PKG_VERSION"),
+    " ( https://github.com/worph/meta-feeder-youtube )"
+);
+
+/// Piped instances, tried in order, rotating past a dead one.
+///
+/// **A pool, not a hostname — measured.** Of four instances tried during the
+/// study, one returned `502`, two `301`-redirected away, and one answered in
+/// ~1 s. That is the failure profile to design for. It is cheap to survive
+/// because we take only the 11-character video id: a dead instance is a retry,
+/// not a broken stream (contrast the *bytes* argument against Piped in
+/// `youtube-source-integration-study.md` §2.2, which does not apply here).
+pub const PIPED_INSTANCES: &[&str] = &[
+    "https://api.piped.private.coffee",
+    "https://pipedapi.kavin.rocks",
+    "https://pipedapi.adminforge.de",
+    "https://api.piped.yt",
+];
+/// Per-request timeout against a Piped instance.
+pub const PIPED_TIMEOUT_SECS: u64 = 15;
+/// Artwork for a YouTube rendition. `hqdefault` exists for every video, unlike
+/// `maxresdefault`, which 404s on anything not uploaded in HD — and a 404 here
+/// is **silent**, gating the record out with no error anywhere.
+pub const YT_THUMB_BASE: &str = "https://i.ytimg.com/vi";
+/// How far a candidate's runtime may sit from the MusicBrainz recording length
+/// and still be the same recording.
+///
+/// ±10 s is what the study measured at, and it is doing two jobs: absorbing
+/// fade-in/out and pre-gap differences between a CD master and an uploaded
+/// Art Track, while staying tight enough to separate a radio edit (249 s) from
+/// an album cut (370 s).
+pub const YT_DURATION_TOLERANCE_SECS: i64 = 10;
+/// Ceiling on Piped `music_songs` candidates examined per track.
+pub const YT_CANDIDATES_PER_TRACK: usize = 8;
+/// How many ranked YouTube references one recording may carry.
+///
+/// §3's mitigation for reference rot: the video **id** is immutable, but its
+/// *availability* is not — takedowns, geo-blocks and embed-disabled uploads all
+/// kill a reference whose CID stays valid forever. One reference per recording
+/// means a takedown kills the track; several let the client fall through.
+///
+/// Three rather than "all of them" because each is a record replicated across
+/// the mesh, and the tail past three is candidates the resolver already ranked
+/// as worse matches.
+pub const MAX_REFS_PER_RECORDING: usize = 3;
+
+// MusicBrainz's published rate ceiling, from the crate that owns that client.
+pub use musicbrainz_feeder::consts::{MB_BURST, MB_RATE_PER_SEC, MB_TIMEOUT_SECS};

feeder-plugin/youtube-feeder/src/lib.rs

@@ -0,0 +1,6 @@
+//! `youtube-feeder` library surface.
+//!
+//! YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded.
+
+pub mod consts;
+pub mod youtube;

feeder-plugin/youtube-feeder/src/main.rs

@@ -0,0 +1,33 @@
+//! `youtube-feeder` — a single-upstream feeder sidecar.
+//!
+//! YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded.
+//!
+//! Upstream: YouTube via Piped instances
+//!
+//! Env:
+//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
+//! - `META_FEEDER_STATE_DIR`   — per-plugin cache root (default `/data/meta-feeder`)
+//! - `RUST_LOG`                — tracing filter (default `info`)
+
+use std::net::SocketAddr;
+
+use meta_feeder_sdk::plugin::FeederPlugin;
+use meta_feeder_sdk::serve_feeders;
+use youtube_feeder::youtube::YouTubePlugin;
+use tracing_subscriber::EnvFilter;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+        .init();
+
+    let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
+        .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
+        .parse()?;
+    let state_dir =
+        std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
+
+    let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(YouTubePlugin::default())];
+    serve_feeders(plugins, state_dir, listen).await
+}

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.89"
+components = ["rustfmt", "clippy"]