feeder rework

.gitattributes

@@ -0,0 +1,6 @@
+# FIX CRLF always when developer has not set
+# Linux/Mac: git config --global core.autocrlf input
+# Windows: git config --global core.autocrlf true
+# Auto detect text files and perform LF normalization
+* text=auto
+* eol=lf

.github/workflows/publish.yml

@@ -0,0 +1,59 @@
+name: Publish image
+
+# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
+#   * every push to main  -> the moving `main` tag
+#   * every `v*` tag       -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
+# The build context is the repo root (the workspace) so the vendored
+# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
+# feeder-plugin/paper-feeder/.
+on:
+  push:
+    branches: [main]
+    tags: ['v*']
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,format=short
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./feeder-plugin/wikicommons-feeder/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Cargo.toml

@@ -0,0 +1,9 @@
+[workspace]
+resolver = "2"
+members = ["feeder-plugin/*"]
+
+[workspace.package]
+edition = "2021"
+rust-version = "1.85"
+license = "MIT"
+repository = "https://github.com/worph/meta-feeder-wikicommons"

README.md

@@ -1 +1,18 @@
-# meta-feeder-wikicommons
+# meta-feeder-wikicommons
+
+Wikimedia Commons media. Open, keyless, no credential.
+
+**Upstream:** Wikimedia Commons (commons.wikimedia.org)
+**Role:** bytes
+
+A MetaMesh gateway feeder — one upstream, one job. It implements
+`FeederPlugin` from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk)
+and is served over the feeder HTTP contract by `serve_feeders`.
+
+```bash
+cargo build --release --bin wikicommons-feeder
+cargo test
+```
+
+Config and scope conventions: see the SDK README and
+`docs/project-architecture/feeder-architecture.md` in the meta-root.

feeder-plugin/wikicommons-feeder/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "wikicommons-feeder"
+version = "0.1.0"
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+publish = false
+
+# Wikimedia Commons media. Open, keyless, no credential.
+# Upstream: Wikimedia Commons (commons.wikimedia.org)
+
+[lib]
+name = "wikicommons_feeder"
+path = "src/lib.rs"
+
+[[bin]]
+name = "wikicommons-feeder"
+path = "src/main.rs"
+
+[dependencies]
+# The SDK is a dependency, not a vendored copy. Pinned by rev until the first
+# tag; see docs/project-architecture/feeder-architecture.md §8.
+meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" }
+
+tokio = { version = "1.40", features = ["full"] }
+axum = { version = "0.7", features = ["json"] }
+anyhow = "1"
+async-trait = "0.1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
+bytes = "1"
+
+[dev-dependencies]
+wiremock = "0.6"
+tempfile = "3"
+tokio = { version = "1.40", features = ["full", "test-util"] }

feeder-plugin/wikicommons-feeder/Dockerfile

@@ -0,0 +1,44 @@
+# syntax=docker/dockerfile:1
+# wikicommons-feeder sidecar. Multi-stage build; build context is this repo's root.
+# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied
+# in — `git` must be present in the builder for cargo to fetch it.
+
+FROM rust:1.89-slim-bookworm AS builder
+WORKDIR /build
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        pkg-config libssl-dev ca-certificates git \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY Cargo.toml rust-toolchain.toml ./
+COPY feeder-plugin/ feeder-plugin/
+
+RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
+    --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
+    --mount=type=cache,id=wikicommons-feeder-target,target=/build/target \
+    cargo build --release --bin wikicommons-feeder \
+    && cp target/release/wikicommons-feeder /wikicommons-feeder
+
+# -----------------------------------------------------------------------------
+FROM debian:bookworm-slim AS runtime
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates curl \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /wikicommons-feeder /usr/local/bin/wikicommons-feeder
+
+RUN useradd --system --uid 10001 --no-create-home metafeeder \
+    && mkdir -p /data/meta-feeder \
+    && chown metafeeder:metafeeder /data/meta-feeder
+
+USER metafeeder
+WORKDIR /data/meta-feeder
+
+EXPOSE 8080
+
+ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
+    META_FEEDER_STATE_DIR=/data/meta-feeder \
+    RUST_LOG=info
+
+ENTRYPOINT ["/usr/local/bin/wikicommons-feeder"]

feeder-plugin/wikicommons-feeder/src/lib.rs

@@ -0,0 +1,4 @@
+//! `wikicommons-feeder` library surface. The binary (`main.rs`) wraps the plugin in
+//! [`meta_feeder_sdk::serve_feeders`].
+
+pub mod wikicommons;

feeder-plugin/wikicommons-feeder/src/main.rs

@@ -0,0 +1,30 @@
+//! `wikicommons-feeder` — a single-upstream feeder sidecar.
+//!
+//! Upstream: Wikimedia Commons (commons.wikimedia.org)
+//!
+//! Env:
+//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
+//! - `META_FEEDER_STATE_DIR`   — per-plugin cache root (default `/data/meta-feeder`)
+//! - `RUST_LOG`                — tracing filter (default `info`)
+
+use std::net::SocketAddr;
+
+use meta_feeder_sdk::{serve_feeders, FeederPlugin};
+use wikicommons_feeder::wikicommons::WikicommonsPlugin;
+use tracing_subscriber::EnvFilter;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+        .init();
+
+    let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
+        .unwrap_or_else(|_| "0.0.0.0:8080".to_string())
+        .parse()?;
+    let state_dir =
+        std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
+
+    let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(WikicommonsPlugin::new())];
+    serve_feeders(plugins, state_dir, listen).await
+}

feeder-plugin/wikicommons-feeder/src/wikicommons.rs

@@ -0,0 +1,989 @@
+//! Wikimedia Commons bridge via the [MediaWiki Action API](https://commons.wikimedia.org/w/api.php).
+//!
+//! - Search: `GET <base>/w/api.php?action=query&generator=search&gsrsearch=<q>&gsrnamespace=6&gsrlimit=N&prop=imageinfo&iiprop=url|mime|size|extmetadata&format=json`
+//!   — one round-trip that returns both search hits and per-file `imageinfo`
+//!   (direct URL, mime, size, license, artist, description). `srnamespace=6`
+//!   pins results to the `File:` namespace so we only get media records.
+//! - Fetch one: `GET <base>/w/api.php?action=query&titles=File:<title>&prop=imageinfo&...`
+//!   — single-record metadata lookup used by `compute_outcomes`.
+//! - File download: the absolute URL from `imageinfo[0].url`
+//!   (typically `https://upload.wikimedia.org/wikipedia/commons/<hash>/<file>`).
+//!
+//! No auth, no API key. Anonymous use is fine for the read paths we use;
+//! the API guide asks for a polite `User-Agent` (already supplied).
+//!
+//! **File-size cap.** Wikimedia Commons hosts files up to several GB
+//! (long-form video). The auto-store path buffers the bytes into RAM
+//! before writing to local meta-core's WebDAV — gigabytes there would
+//! OOM the gateway peer. This plugin refuses anything over
+//! [`MAX_FILE_BYTES`] (50 MB) with a permanent error, leaving streaming
+//! support for a later feature.
+//!
+//! See `docs/gateway-feature.md` §4 for the v0 plugin role.
+
+use std::collections::{BTreeMap, HashMap};
+use std::path::Path;
+
+use async_trait::async_trait;
+use serde::Deserialize;
+
+use meta_feeder_sdk::common;
+use meta_feeder_sdk::cache::MidhashCache;
+use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, GatewayQuery, HashOutcome};
+use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
+
+/// Wikimedia Commons API root. Overridable by tests to point at a
+/// `wiremock::MockServer`.
+const DEFAULT_BASE_URL: &str = "https://commons.wikimedia.org";
+
+/// HTTP timeout per upstream call. Generous to cover the occasional
+/// upload-server cold-cache fetch for larger media files (image/video).
+const HTTP_TIMEOUT_SECS: u64 = 60;
+
+/// Polite identification — Wikimedia's API policy requires a meaningful
+/// User-Agent so they can rate-limit per client.
+const USER_AGENT: &str = concat!(
+    "meta-share/",
+    env!("CARGO_PKG_VERSION"),
+    " (gateway:wikicommons)"
+);
+
+/// Hard ceiling on the file size we'll pull through `compute_outcomes`.
+/// 50 MB comfortably covers all common-image, most audio, and small
+/// video records on Commons; rejects the multi-GB long-form videos
+/// that would OOM the gateway peer's buffer-to-RAM auto-store path.
+/// Streaming support relaxes this when the auto-store pipeline goes
+/// chunked (v0.3 territory).
+const MAX_FILE_BYTES: u64 = 50 * 1024 * 1024;
+
+/// Thumbnail width (in pixels) requested via MediaWiki's `iiurlwidth`
+/// parameter. 400 keeps the resulting JPEG/PNG in the ~30–80 KB range
+/// per image — small enough to fetch + seed per search result without
+/// blowing the search-latency budget, large enough that an inline card
+/// preview doesn't look pixelated.
+const PREVIEW_WIDTH_PX: u32 = 400;
+
+/// Wikimedia Commons gateway plugin. Same structural shape as the other
+/// fetch-capable plugins.
+pub struct WikicommonsPlugin {
+    http: reqwest::Client,
+    base_url: String,
+    cache: Option<MidhashCache>,
+}
+
+impl Default for WikicommonsPlugin {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl WikicommonsPlugin {
+    pub fn new() -> Self {
+        Self::with_base_url(DEFAULT_BASE_URL.to_string())
+    }
+
+    /// Override the API root for tests. The mock serves both
+    /// `/w/api.php` (search + imageinfo) and the upload paths returned
+    /// in the `imageinfo[0].url` field — the test fixture builds those
+    /// URLs against `server.uri()` so the same host handles all calls.
+    pub fn with_base_url(base_url: String) -> Self {
+        let http = common::build_http_client(HTTP_TIMEOUT_SECS, USER_AGENT, None);
+        Self {
+            http,
+            base_url,
+            cache: None,
+        }
+    }
+
+    fn cache(&self) -> Result<&MidhashCache, GatewayError> {
+        common::require_cache(self.cache.as_ref(), "wikicommons")
+    }
+
+    /// Build the `/w/api.php` URL for one of our two query shapes:
+    /// - search: `extra` = `generator=search&gsrsearch=<q>&gsrnamespace=6&gsrlimit=<n>`
+    /// - single: `extra` = `titles=File:<title>`
+    ///
+    /// The common bits (`action=query&prop=imageinfo&iiprop=...&format=json`)
+    /// live here so the two callers stay short.
+    ///
+    /// `iiurlwidth=<PREVIEW_WIDTH_PX>` asks MediaWiki to also include a
+    /// thumbnail URL (`thumburl`) in each `imageinfo[0]`. `handle_query`
+    /// surfaces that as a raw `preview_url` field; the gateway core fetches,
+    /// seeds, and rewrites it to a `preview` cid (plan §2c). Audio/video
+    /// records lack a thumbnail (MediaWiki omits `thumburl` for non-image
+    /// mimes); those ship without a `preview_url`.
+    fn api_url(&self, extra: &str) -> String {
+        format!(
+            "{}/w/api.php?action=query&prop=imageinfo&iiprop=url%7Cmime%7Csize%7Cextmetadata&iiurlwidth={}&format=json&formatversion=1&{}",
+            self.base_url.trim_end_matches('/'),
+            PREVIEW_WIDTH_PX,
+            extra
+        )
+    }
+
+    async fn fetch_api(&self, url: &str) -> Result<MediawikiResponse, GatewayError> {
+        let resp = self
+            .http
+            .get(url)
+            .send()
+            .await
+            .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?;
+        common::map_status(&resp)?;
+        resp.json::<MediawikiResponse>()
+            .await
+            .map_err(|e| GatewayError::Permanent(format!("decode mediawiki response: {e}")))
+    }
+
+    /// Free-text search on the File: namespace via `generator=search`.
+    /// Single round-trip returns search hits and their `imageinfo` — no
+    /// per-record lookup needed at search time.
+    async fn search_files(
+        &self,
+        query: &str,
+        max_results: usize,
+    ) -> Result<Vec<MediawikiPage>, GatewayError> {
+        let url = self.api_url(&format!(
+            "generator=search&gsrsearch={}&gsrnamespace=6&gsrlimit={}",
+            common::urlencode(query),
+            max_results
+        ));
+        let body = self.fetch_api(&url).await?;
+        Ok(body.into_ordered_pages())
+    }
+
+    /// Single-record lookup by canonical title. `record_id` is the title
+    /// **without** the `File:` prefix (that's how we record it). We add
+    /// the prefix back before the API call.
+    async fn fetch_page(&self, record_id: &str) -> Result<MediawikiPage, GatewayError> {
+        let url = self.api_url(&format!("titles=File:{}", common::urlencode(record_id)));
+        let body = self.fetch_api(&url).await?;
+        let pages = body.into_ordered_pages();
+        let page = pages.into_iter().next().ok_or(GatewayError::NotFound)?;
+        // MediaWiki returns a synthetic `{"missing": ""}` page for
+        // unknown titles. Detect by absence of pageid + imageinfo.
+        if page.imageinfo.as_deref().unwrap_or(&[]).is_empty() && page.pageid.is_none() {
+            return Err(GatewayError::NotFound);
+        }
+        Ok(page)
+    }
+
+    /// Pull the binary file bytes from the absolute URL Wikimedia returned
+    /// in `imageinfo[0].url` (full-file fetch for `compute_outcomes`) or
+    /// `imageinfo[0].thumburl` (thumbnail fetch for `handle_query` →
+    /// preview seed). The URL points at `upload.wikimedia.org` for real
+    /// Commons content; the test fixture replaces it with the mock
+    /// server's URL, so we just trust whatever the API gave us.
+    async fn fetch_file(&self, url: &str) -> Result<bytes::Bytes, GatewayError> {
+        common::fetch_bytes(&self.http, url).await
+    }
+}
+
+#[async_trait]
+impl FeederPlugin for WikicommonsPlugin {
+    fn upstream_id(&self) -> &'static str {
+        "wikicommons"
+    }
+
+    fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
+        self.cache = Some(common::open_midhash_cache(cache_dir, "wikicommons")?);
+        Ok(())
+    }
+
+    async fn handle_query(
+        &self,
+        query: &GatewayQuery,
+        max_results: usize,
+    ) -> Result<Vec<DiscoveryRecord>, GatewayError> {
+        // Layer A early-return: a `fileType:` / `contentKind:` filter
+        // wholly outside this plugin's served set can't ever match,
+        // so don't burn an upstream API call. Wikicommons serves
+        // image/audio/video/document/other and no content kinds, so
+        // this only rejects a query like `contentKind:movie` or a
+        // bespoke `fileType:archive` (none of which Commons answers).
+        if !meta_feeder_sdk::query_eval::query_accepts_plugin(
+            query,
+            self.served_file_types(),
+            self.served_content_kinds(),
+        ) {
+            return Ok(Vec::new());
+        }
+        // Layer A pushdown: if the user filtered by `fileType`, fold
+        // the requested types into MediaWiki's `filetype:` CirrusSearch
+        // keyword so the upstream returns the right kind in the first
+        // place. Without this, broad-spectrum Commons returns jpgs for
+        // a `fileType:video` query and the dispatcher post-filter has
+        // to throw most of them away. See `mediawiki_filetype_clause`
+        // for the meta-vocabulary → CirrusSearch value mapping.
+        let base_q = query.free_text_or_star();
+        let composed_q;
+        let q = match mediawiki_filetype_clause(query) {
+            Some(clause) => {
+                composed_q = format!("{base_q} {clause}");
+                composed_q.as_str()
+            }
+            None => base_q,
+        };
+        let pages = self.search_files(q, max_results).await?;
+        // Emit each result's thumbnail URL as a raw `preview_url` field; the
+        // gateway core fetches it, seeds the bytes into its bitswap blockstore,
+        // and rewrites it to a content-addressed `preview` cid (plan §2c) — a
+        // feeder can't reach the core blockstore. Non-image records (audio /
+        // video / pdf) carry no thumbnail and ship preview-less.
+        let records: Vec<DiscoveryRecord> = pages
+            .into_iter()
+            .filter_map(|p| {
+                let thumb_url = p
+                    .imageinfo
+                    .as_ref()
+                    .and_then(|v| v.first())
+                    .and_then(|info| info.thumburl.clone());
+                let mut rec = into_discovery_record(p)?;
+                if let Some(u) = thumb_url {
+                    rec.fields.insert("preview_url".to_string(), u);
+                }
+                Some(rec)
+            })
+            .take(max_results)
+            .collect();
+        Ok(records)
+    }
+
+    async fn compute_outcomes(&self, record_id: &str) -> Result<Vec<HashOutcome>, GatewayError> {
+        let cache = self.cache()?;
+        if let Some(hit) = common::cached_outcome(cache, record_id, "wikicommons")? {
+            return Ok(hit);
+        }
+
+        let page = self.fetch_page(record_id).await?;
+        let info = page
+            .imageinfo
+            .as_ref()
+            .and_then(|v| v.first())
+            .ok_or_else(|| {
+                GatewayError::Permanent(format!(
+                    "wikicommons record `{record_id}` has no imageinfo"
+                ))
+            })?;
+        // Refuse oversized files before downloading so a multi-GB video
+        // doesn't OOM the gateway peer. The 50 MB ceiling covers the
+        // overwhelming majority of Commons records (images + small
+        // audio/video); streaming support relaxes this later.
+        if let Some(size) = info.size {
+            if size > MAX_FILE_BYTES {
+                return Err(GatewayError::Permanent(format!(
+                    "wikicommons record `{record_id}` is {size} bytes — exceeds gateway plugin's \
+                     {MAX_FILE_BYTES}-byte ceiling. Streaming auto-store would lift this; not in v0."
+                )));
+            }
+        }
+        let url = &info.url;
+        let bytes = self.fetch_file(url).await?;
+        // Defence-in-depth: even if `size` was None / missing in the
+        // imageinfo, the downloaded body shouldn't exceed the same cap.
+        if bytes.len() as u64 > MAX_FILE_BYTES {
+            return Err(GatewayError::Permanent(format!(
+                "wikicommons record `{record_id}` body was {} bytes — exceeds {MAX_FILE_BYTES} ceiling",
+                bytes.len()
+            )));
+        }
+        let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes);
+
+        common::store_midhash(cache, record_id, "wikicommons", &cid);
+
+        let file_extension = file_extension_for(record_id);
+        let record = into_discovery_record(page).ok_or_else(|| {
+            // We just successfully fetched a page; this should not
+            // happen unless imageinfo went missing between API calls.
+            GatewayError::Permanent(format!(
+                "wikicommons record `{record_id}` lost imageinfo between calls"
+            ))
+        })?;
+        Ok(common::single_outcome(cid, bytes, record, file_extension))
+    }
+
+    fn health(&self) -> PluginHealth {
+        if self.cache.is_some() {
+            PluginHealth::Ok
+        } else {
+            PluginHealth::Degraded {
+                reason: "configure() not yet called".to_string(),
+            }
+        }
+    }
+
+    fn served_file_types(&self) -> &'static [&'static str] {
+        // Wikicommons holds heterogeneous mime types — `media_type_for`
+        // buckets each result into one of meta-core's `fileType` values.
+        // `other` covers everything we can't classify.
+        &["image", "audio", "video", "document", "other"]
+    }
+
+    fn served_content_kinds(&self) -> &'static [&'static str] {
+        // No semantic refinement yet — wikicommons records carry only
+        // the coarse mime bucket. Future: photo / illustration / gif
+        // classification could land here.
+        &[]
+    }
+}
+
+// -- MediaWiki JSON shapes (subset we use) ----------------------------------
+
+#[derive(Debug, Default, Deserialize)]
+struct MediawikiResponse {
+    #[serde(default)]
+    query: Option<MediawikiQuery>,
+}
+
+impl MediawikiResponse {
+    /// Pages come back as a `pages` map keyed by stringified pageid.
+    /// `generator=search` adds an `index` field to each page reflecting
+    /// the original search ranking — we sort on that so the caller sees
+    /// hits in relevance order. Single-record lookups have no `index`
+    /// and reach `into_ordered_pages` with at most one entry anyway.
+    fn into_ordered_pages(self) -> Vec<MediawikiPage> {
+        let Some(query) = self.query else {
+            return Vec::new();
+        };
+        let mut pages: Vec<MediawikiPage> = query.pages.into_values().collect();
+        pages.sort_by_key(|p| p.index.unwrap_or(u32::MAX));
+        pages
+    }
+}
+
+#[derive(Debug, Default, Deserialize)]
+struct MediawikiQuery {
+    #[serde(default)]
+    pages: HashMap<String, MediawikiPage>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(default)]
+struct MediawikiPage {
+    pageid: Option<u64>,
+    title: Option<String>,
+    /// Search-rank position (1-based) — only populated when the page
+    /// came from `generator=search`. Lower = more relevant; we sort on
+    /// this so search results don't get scrambled by `HashMap` order.
+    index: Option<u32>,
+    imageinfo: Option<Vec<MediawikiImageInfo>>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct MediawikiImageInfo {
+    url: String,
+    #[serde(default)]
+    mime: Option<String>,
+    #[serde(default)]
+    size: Option<u64>,
+    /// Scaled-down preview URL — only populated when the request
+    /// included `iiurlwidth=<n>` AND the file is an image. For
+    /// audio/video records MediaWiki omits this field even when
+    /// `iiurlwidth` was set (no thumbnail exists), and we ship the
+    /// record without a `preview` cid.
+    #[serde(default)]
+    thumburl: Option<String>,
+    /// Wikimedia's extmetadata block — license, artist, description,
+    /// date. Each entry is `{ "value": "…", "source": "…" }`; we keep
+    /// only the value via the inner struct.
+    #[serde(default)]
+    extmetadata: Option<MediawikiExtMetadata>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(default, rename_all = "PascalCase")]
+struct MediawikiExtMetadata {
+    artist: Option<ExtValue>,
+    license_short_name: Option<ExtValue>,
+    image_description: Option<ExtValue>,
+    date_time_original: Option<ExtValue>,
+    credit: Option<ExtValue>,
+}
+
+#[derive(Debug, Default, Deserialize)]
+#[serde(default)]
+struct ExtValue {
+    value: String,
+}
+
+/// Map a MediaWiki `page` (with imageinfo) into a `DiscoveryRecord`.
+/// Returns `None` when the page has no imageinfo entry — happens for
+/// deleted files where the page still exists, and for the synthetic
+/// `missing=""` page MediaWiki emits for unknown titles.
+fn into_discovery_record(page: MediawikiPage) -> Option<DiscoveryRecord> {
+    let title = page.title?;
+    // `record_id` is the title with the `File:` prefix stripped — that's
+    // how the rest of the gateway tier refers to it (URL paths, library
+    // entries, the synthetic `gateway:wikicommons:<id>` CID).
+    let record_id = title
+        .strip_prefix("File:")
+        .map(str::to_string)
+        .unwrap_or(title.clone());
+    let info = page.imageinfo?.into_iter().next()?;
+    let mut fields: BTreeMap<String, String> = BTreeMap::new();
+    fields.insert("title".to_string(), record_id.clone());
+    fields.insert("fileType".to_string(), media_type_for(info.mime.as_deref()));
+    fields.insert(
+        "sourceUrl".to_string(),
+        format!(
+            "https://commons.wikimedia.org/wiki/{}",
+            encode_title_for_url(&title)
+        ),
+    );
+    fields.insert("fileName".to_string(), record_id.clone());
+    fields.insert(upstream_id_field("wikicommons"), record_id.clone());
+    // The `preview` field (a content-addressed cid for an inline
+    // thumbnail) is NOT set here — `handle_query` seeds the bytes into
+    // the gateway's blockstore in parallel and attaches the cid
+    // post-hoc. We deliberately don't emit an upstream URL field; the
+    // design is "every byte reference is a cid", so dashboards never
+    // need to know what `upload.wikimedia.org` is.
+    if let Some(mime) = info.mime.as_deref() {
+        fields.insert("mime".to_string(), mime.to_string());
+    }
+    if let Some(ext) = file_extension_for(&record_id) {
+        fields.insert("format".to_string(), ext);
+    }
+    if let Some(size) = info.size {
+        fields.insert("size".to_string(), size.to_string());
+    }
+    if let Some(em) = info.extmetadata {
+        if let Some(v) = em.artist {
+            fields.insert("artist".to_string(), v.value);
+        }
+        if let Some(v) = em.license_short_name {
+            fields.insert("license".to_string(), v.value);
+        }
+        if let Some(v) = em.image_description {
+            fields.insert("description".to_string(), v.value);
+        }
+        if let Some(v) = em.date_time_original {
+            fields.insert("dateOriginal".to_string(), v.value);
+        }
+        if let Some(v) = em.credit {
+            fields.insert("credit".to_string(), v.value);
+        }
+    }
+    Some(DiscoveryRecord {
+        upstream_id: "wikicommons".to_string(),
+        record_id,
+        fields,
+    })
+}
+
+/// Bucket a MediaWiki mime into a meta-core `fileType` value
+/// (vocabulary defined in
+/// [docs/project-architecture/metadata-keys.md → `fileType`]).
+/// `image/jpeg` → "image", `audio/ogg` → "audio", `video/webm` → "video",
+/// `application/pdf` → "document", everything else → "other".
+fn media_type_for(mime: Option<&str>) -> String {
+    let Some(mime) = mime else {
+        return "other".to_string();
+    };
+    if mime.starts_with("image/") {
+        "image".to_string()
+    } else if mime.starts_with("audio/") {
+        "audio".to_string()
+    } else if mime.starts_with("video/") {
+        "video".to_string()
+    } else if mime.starts_with("application/pdf") {
+        "document".to_string()
+    } else {
+        "other".to_string()
+    }
+}
+
+/// File extension from the canonical title. Mirror of arxiv / pubmed's
+/// per-plugin hardcoded `"pdf"`, but Commons records carry heterogeneous
+/// types — derive from the title (which Wikimedia normalises to include
+/// the extension). Returns the lowercased extension without the dot, or
+/// `None` if the title has no recognisable extension.
+fn file_extension_for(record_id: &str) -> Option<String> {
+    let dot = record_id.rfind('.')?;
+    let ext = &record_id[dot + 1..];
+    if ext.is_empty() || ext.len() > 5 || !ext.chars().all(|c| c.is_ascii_alphanumeric()) {
+        return None;
+    }
+    Some(ext.to_ascii_lowercase())
+}
+
+/// Percent-encode a MediaWiki title for the wiki page URL.
+/// Wikimedia uses underscores for spaces in URLs (the API normalises
+/// spaces → underscores anyway), and percent-encodes the rest. We keep
+/// it minimal: replace spaces, percent-encode everything else outside
+/// the unreserved set.
+fn encode_title_for_url(title: &str) -> String {
+    let with_underscores: String = title
+        .chars()
+        .map(|c| if c == ' ' { '_' } else { c })
+        .collect();
+    let mut out = String::with_capacity(with_underscores.len());
+    for c in with_underscores.chars() {
+        match c {
+            'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' | '/' | ':' => out.push(c),
+            _ => {
+                let mut buf = [0u8; 4];
+                for b in c.encode_utf8(&mut buf).as_bytes() {
+                    out.push_str(&format!("%{:02X}", b));
+                }
+            }
+        }
+    }
+    out
+}
+
+/// Percent-encode a value for the query string. Same safe-set the other
+/// plugins use; covers MediaWiki's search query syntax.
+/// Map `query.filters["fileType"]` (our cohort vocabulary) into a
+/// MediaWiki CirrusSearch `filetype:` clause that Commons' search
+/// understands. Returns `None` when the query has no `fileType:`
+/// filter, so the caller can leave the upstream query unmodified.
+///
+/// MediaWiki vocabulary (`filetype:` keyword on Commons): `bitmap`,
+/// `drawing`, `audio`, `video`, `office`, `unknown`. Mapping from our
+/// coarse `fileType` enum:
+///
+/// | meta-cohort `fileType` | MediaWiki `filetype:` value(s) |
+/// |---|---|
+/// | `image`    | `bitmap`, `drawing`               |
+/// | `audio`    | `audio`                           |
+/// | `video`    | `video`                           |
+/// | `document` | `office`                          |
+/// | (other)    | dropped — Commons has no concept  |
+///
+/// CirrusSearch syntax for OR-ing filetype values is **pipe-separated
+/// inside a single keyword**: `filetype:bitmap|drawing`. Using
+/// explicit `OR` between two `filetype:` keywords parses as an
+/// impossible AND (a file can't be both `bitmap` AND `drawing`), so
+/// emit the pipe form unconditionally.
+fn mediawiki_filetype_clause(query: &GatewayQuery) -> Option<String> {
+    let requested = query.filters.get("fileType")?;
+    if requested.is_empty() {
+        return None;
+    }
+    let mut mapped: Vec<&'static str> = Vec::new();
+    for v in requested {
+        let extra: &[&'static str] = match v.to_ascii_lowercase().as_str() {
+            "image" => &["bitmap", "drawing"],
+            "audio" => &["audio"],
+            "video" => &["video"],
+            "document" => &["office"],
+            // `other`, `subtitle`, `archive`, … — no Commons equivalent.
+            // Skip rather than emit a syntactically-broken clause; the
+            // dispatcher's post-filter (Layer B) will drop anything
+            // that comes back with the wrong fileType anyway.
+            _ => &[],
+        };
+        for e in extra {
+            if !mapped.iter().any(|m| m == e) {
+                mapped.push(e);
+            }
+        }
+    }
+    if mapped.is_empty() {
+        return None;
+    }
+    Some(format!("filetype:{}", mapped.join("|")))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Sample `generator=search` response with two image hits.
+    /// Real Wikimedia API output, trimmed.
+    const SAMPLE_SEARCH: &str = r#"{
+      "batchcomplete": "",
+      "query": {
+        "pages": {
+          "1": {
+            "pageid": 1,
+            "ns": 6,
+            "title": "File:Sunrise over the sea.jpg",
+            "index": 1,
+            "imageinfo": [
+              {
+                "url": "https://upload.wikimedia.org/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg",
+                "mime": "image/jpeg",
+                "size": 234567,
+                "extmetadata": {
+                  "Artist":           { "value": "Jane Doe", "source": "commons" },
+                  "LicenseShortName": { "value": "CC BY-SA 4.0", "source": "commons" },
+                  "ImageDescription": { "value": "A sunrise photographed off the coast.", "source": "commons" },
+                  "DateTimeOriginal": { "value": "2023-08-12 06:14:00", "source": "exif" },
+                  "Credit":           { "value": "Own work", "source": "commons" }
+                }
+              }
+            ]
+          },
+          "2": {
+            "pageid": 2,
+            "ns": 6,
+            "title": "File:Echinacea_purpurea.jpg",
+            "index": 2,
+            "imageinfo": [
+              {
+                "url": "https://upload.wikimedia.org/wikipedia/commons/x/yz/Echinacea_purpurea.jpg",
+                "mime": "image/jpeg",
+                "size": 145000
+              }
+            ]
+          }
+        }
+      }
+    }"#;
+
+    fn empty_gateway_query() -> GatewayQuery {
+        GatewayQuery {
+            raw_text: String::new(),
+            free_text: String::new(),
+            filters: std::collections::BTreeMap::new(),
+            ranges: Vec::new(),
+            negations: Vec::new(),
+        }
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_returns_none_for_unfiltered_query() {
+        assert_eq!(mediawiki_filetype_clause(&empty_gateway_query()), None);
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_maps_video() {
+        let mut q = empty_gateway_query();
+        q.filters.insert("fileType".into(), vec!["video".into()]);
+        assert_eq!(
+            mediawiki_filetype_clause(&q),
+            Some("filetype:video".to_string())
+        );
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_maps_image_to_bitmap_and_drawing() {
+        let mut q = empty_gateway_query();
+        q.filters.insert("fileType".into(), vec!["image".into()]);
+        assert_eq!(
+            mediawiki_filetype_clause(&q),
+            Some("filetype:bitmap|drawing".to_string())
+        );
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_ors_multiple_requested_types() {
+        let mut q = empty_gateway_query();
+        q.filters
+            .insert("fileType".into(), vec!["video".into(), "audio".into()]);
+        assert_eq!(
+            mediawiki_filetype_clause(&q),
+            Some("filetype:video|audio".to_string())
+        );
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_dedupes_overlapping_mappings() {
+        let mut q = empty_gateway_query();
+        // image → [bitmap, drawing]; document → [office]; both kept.
+        q.filters.insert(
+            "fileType".into(),
+            vec!["image".into(), "document".into(), "image".into()],
+        );
+        assert_eq!(
+            mediawiki_filetype_clause(&q),
+            Some("filetype:bitmap|drawing|office".to_string())
+        );
+    }
+
+    #[test]
+    fn mediawiki_filetype_clause_drops_unmappable_values() {
+        let mut q = empty_gateway_query();
+        q.filters
+            .insert("fileType".into(), vec!["archive".into(), "subtitle".into()]);
+        assert_eq!(mediawiki_filetype_clause(&q), None);
+    }
+
+    #[test]
+    fn parses_search_response_in_index_order() {
+        let resp: MediawikiResponse = serde_json::from_str(SAMPLE_SEARCH).expect("parse");
+        let pages = resp.into_ordered_pages();
+        assert_eq!(pages.len(), 2);
+        assert_eq!(
+            pages[0].title.as_deref(),
+            Some("File:Sunrise over the sea.jpg")
+        );
+        assert_eq!(
+            pages[1].title.as_deref(),
+            Some("File:Echinacea_purpurea.jpg")
+        );
+    }
+
+    #[test]
+    fn into_discovery_record_emits_required_fields() {
+        let resp: MediawikiResponse = serde_json::from_str(SAMPLE_SEARCH).unwrap();
+        let pages = resp.into_ordered_pages();
+        let rec = into_discovery_record(pages.into_iter().next().unwrap()).expect("rec");
+        assert_eq!(rec.upstream_id, "wikicommons");
+        assert_eq!(rec.record_id, "Sunrise over the sea.jpg");
+        // §6.6 conventions: fileType (mime bucket) + no contentKind (no
+        // semantic refinement on wikicommons records yet).
+        assert_eq!(
+            rec.fields.get("fileType").map(String::as_str),
+            Some("image")
+        );
+        assert!(!rec.fields.contains_key("contentKind"));
+        assert_eq!(rec.fields.get("format").map(String::as_str), Some("jpg"));
+        assert_eq!(
+            rec.fields.get("sourceUrl").map(String::as_str),
+            Some("https://commons.wikimedia.org/wiki/File:Sunrise_over_the_sea.jpg")
+        );
+        assert_eq!(
+            rec.fields.get("fileName").map(String::as_str),
+            Some("Sunrise over the sea.jpg")
+        );
+        // Canonical `<upstream_id>id` field.
+        assert_eq!(
+            rec.fields.get("wikicommonsid").map(String::as_str),
+            Some("Sunrise over the sea.jpg")
+        );
+        // Extmetadata surface.
+        assert_eq!(
+            rec.fields.get("artist").map(String::as_str),
+            Some("Jane Doe")
+        );
+        assert_eq!(
+            rec.fields.get("license").map(String::as_str),
+            Some("CC BY-SA 4.0")
+        );
+        assert_eq!(
+            rec.fields.get("description").map(String::as_str),
+            Some("A sunrise photographed off the coast.")
+        );
+    }
+
+    #[test]
+    fn media_type_buckets_match_mime() {
+        assert_eq!(media_type_for(Some("image/jpeg")), "image");
+        assert_eq!(media_type_for(Some("audio/ogg")), "audio");
+        assert_eq!(media_type_for(Some("video/webm")), "video");
+        assert_eq!(media_type_for(Some("application/pdf")), "document");
+        assert_eq!(media_type_for(Some("application/octet-stream")), "other");
+        assert_eq!(media_type_for(None), "other");
+    }
+
+    #[test]
+    fn file_extension_for_handles_common_shapes() {
+        assert_eq!(file_extension_for("Sunrise.jpg").as_deref(), Some("jpg"));
+        assert_eq!(file_extension_for("Foo.OGG").as_deref(), Some("ogg"));
+        assert_eq!(file_extension_for("Foo.webm").as_deref(), Some("webm"));
+        // Too long to be an extension.
+        assert!(file_extension_for("Foo.thisisalonglongtail").is_none());
+        // No extension at all.
+        assert!(file_extension_for("BareName").is_none());
+        // Non-alphanumeric trailing — refused.
+        assert!(file_extension_for("Foo.tar.gz?x").is_none());
+    }
+
+    #[test]
+    fn encode_title_swaps_spaces_for_underscores() {
+        assert_eq!(
+            encode_title_for_url("File:Sunrise over the sea.jpg"),
+            "File:Sunrise_over_the_sea.jpg"
+        );
+        assert_eq!(
+            encode_title_for_url("File:Caf\u{00e9}.jpg"),
+            "File:Caf%C3%A9.jpg"
+        );
+    }
+
+    // -- HTTP integration tests against a wiremock server ----------------
+
+    use wiremock::matchers::{method, path, query_param};
+    use wiremock::{Mock, MockServer, ResponseTemplate};
+
+    fn configured_plugin_against(server: &MockServer) -> (WikicommonsPlugin, tempfile::TempDir) {
+        let dir = tempfile::tempdir().expect("tempdir");
+        let mut plugin = WikicommonsPlugin::with_base_url(server.uri());
+        plugin.configure(dir.path()).expect("configure");
+        (plugin, dir)
+    }
+
+    #[tokio::test]
+    async fn handle_query_maps_search_to_discovery_records() {
+        let server = MockServer::start().await;
+        Mock::given(method("GET"))
+            .and(path("/w/api.php"))
+            .and(query_param("generator", "search"))
+            .and(query_param("gsrsearch", "sunrise"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_string(SAMPLE_SEARCH)
+                    .insert_header("content-type", "application/json"),
+            )
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let records = plugin
+            .handle_query(&GatewayQuery::from_free_text("sunrise"), 10)
+            .await
+            .expect("handle_query");
+        assert_eq!(records.len(), 2);
+        assert_eq!(records[0].record_id, "Sunrise over the sea.jpg");
+    }
+
+    #[tokio::test]
+    async fn handle_query_truncates_to_max_results() {
+        let server = MockServer::start().await;
+        Mock::given(method("GET"))
+            .and(path("/w/api.php"))
+            .respond_with(ResponseTemplate::new(200).set_body_string(SAMPLE_SEARCH))
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let records = plugin
+            .handle_query(&GatewayQuery::from_free_text("sunrise"), 1)
+            .await
+            .expect("handle_query");
+        assert_eq!(records.len(), 1);
+    }
+
+    #[tokio::test]
+    async fn compute_outcomes_fetches_file_and_caches() {
+        let server = MockServer::start().await;
+        // Build a single-record response that points at the mock's own
+        // host, so the file fetch is intercepted too.
+        let img_bytes = b"\xFF\xD8\xFF mock jpeg body for hashing\n".to_vec();
+        let expected_cid = meta_feeder_sdk::hash::compute_ipfs_cid(&img_bytes);
+        let single = format!(
+            r#"{{
+              "batchcomplete": "",
+              "query": {{
+                "pages": {{
+                  "1": {{
+                    "pageid": 1,
+                    "ns": 6,
+                    "title": "File:Sunrise over the sea.jpg",
+                    "imageinfo": [
+                      {{
+                        "url": "{base}/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg",
+                        "mime": "image/jpeg",
+                        "size": {sz}
+                      }}
+                    ]
+                  }}
+                }}
+              }}
+            }}"#,
+            base = server.uri(),
+            sz = img_bytes.len()
+        );
+        Mock::given(method("GET"))
+            .and(path("/w/api.php"))
+            .and(query_param("titles", "File:Sunrise over the sea.jpg"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_string(single)
+                    .insert_header("content-type", "application/json"),
+            )
+            .mount(&server)
+            .await;
+        Mock::given(method("GET"))
+            .and(path("/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_bytes(img_bytes.clone())
+                    .insert_header("content-type", "image/jpeg"),
+            )
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let outcome = plugin
+            .compute_outcomes("Sunrise over the sea.jpg")
+            .await
+            .expect("compute")
+            .into_iter()
+            .next()
+            .expect("single outcome");
+        assert_eq!(outcome.hash.as_str(), expected_cid);
+        assert_eq!(outcome.bytes.as_deref(), Some(img_bytes.as_slice()));
+        assert_eq!(outcome.file_extension.as_deref(), Some("jpg"));
+        let rec = outcome.record.expect("record present on fresh compute");
+        assert_eq!(rec.record_id, "Sunrise over the sea.jpg");
+        assert_eq!(
+            rec.fields.get("fileType").map(String::as_str),
+            Some("image")
+        );
+    }
+
+    #[tokio::test]
+    async fn compute_outcomes_cache_hit_skips_http() {
+        let server = MockServer::start().await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        plugin
+            .cache
+            .as_ref()
+            .unwrap()
+            .put_midhash("Sunrise.jpg", "bafyCACHED")
+            .unwrap();
+        let outcome = plugin
+            .compute_outcomes("Sunrise.jpg")
+            .await
+            .expect("cache hit")
+            .into_iter()
+            .next()
+            .expect("single outcome");
+        assert_eq!(outcome.hash.as_str(), "bafyCACHED");
+        assert!(outcome.bytes.is_none());
+        assert!(outcome.record.is_none());
+    }
+
+    #[tokio::test]
+    async fn compute_outcomes_refuses_oversized_records() {
+        let server = MockServer::start().await;
+        let too_big = MAX_FILE_BYTES + 1;
+        let single = format!(
+            r#"{{
+              "batchcomplete": "",
+              "query": {{
+                "pages": {{
+                  "1": {{
+                    "pageid": 1,
+                    "ns": 6,
+                    "title": "File:Huge.webm",
+                    "imageinfo": [
+                      {{ "url": "{base}/foo.webm", "mime": "video/webm", "size": {sz} }}
+                    ]
+                  }}
+                }}
+              }}
+            }}"#,
+            base = server.uri(),
+            sz = too_big
+        );
+        Mock::given(method("GET"))
+            .and(path("/w/api.php"))
+            .respond_with(ResponseTemplate::new(200).set_body_string(single))
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let err = plugin
+            .compute_outcomes("Huge.webm")
+            .await
+            .expect_err("oversized");
+        assert!(matches!(err, GatewayError::Permanent(_)), "got {err:?}");
+    }
+
+    #[tokio::test]
+    async fn upstream_404_maps_to_not_found() {
+        let server = MockServer::start().await;
+        Mock::given(method("GET"))
+            .and(path("/w/api.php"))
+            .respond_with(ResponseTemplate::new(404))
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let err = plugin
+            .compute_outcomes("Does-not-exist.jpg")
+            .await
+            .expect_err("expected NotFound");
+        assert!(matches!(err, GatewayError::NotFound), "got {err:?}");
+    }
+}

rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "1.89"
+components = ["rustfmt", "clippy"]