diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..645e848 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/paper-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/wikicommons-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ecc351e --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1946 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +source = "git+https://github.com/worph/meta-feeder-sdk?rev=d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4#d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "wikicommons-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "meta-feeder-sdk", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..4bb9d4a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-wikicommons" diff --git a/README.md b/README.md index 6436ea0..f65e785 100644 --- a/README.md +++ b/README.md @@ -1 +1,18 @@ -# meta-feeder-wikicommons \ No newline at end of file +# meta-feeder-wikicommons + +Wikimedia Commons media. Open, keyless, no credential. + +**Upstream:** Wikimedia Commons (commons.wikimedia.org) +**Role:** bytes + +A MetaMesh gateway feeder — one upstream, one job. It implements +`FeederPlugin` from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk) +and is served over the feeder HTTP contract by `serve_feeders`. + +```bash +cargo build --release --bin wikicommons-feeder +cargo test +``` + +Config and scope conventions: see the SDK README and +`docs/project-architecture/feeder-architecture.md` in the meta-root. diff --git a/feeder-plugin/wikicommons-feeder/Cargo.toml b/feeder-plugin/wikicommons-feeder/Cargo.toml new file mode 100644 index 0000000..d1eb184 --- /dev/null +++ b/feeder-plugin/wikicommons-feeder/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "wikicommons-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Wikimedia Commons media. Open, keyless, no credential. +# Upstream: Wikimedia Commons (commons.wikimedia.org) + +[lib] +name = "wikicommons_feeder" +path = "src/lib.rs" + +[[bin]] +name = "wikicommons-feeder" +path = "src/main.rs" + +[dependencies] +# The SDK is a dependency, not a vendored copy. Pinned by rev until the first +# tag; see docs/project-architecture/feeder-architecture.md §8. +meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +bytes = "1" + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/wikicommons-feeder/Dockerfile b/feeder-plugin/wikicommons-feeder/Dockerfile new file mode 100644 index 0000000..fc80df4 --- /dev/null +++ b/feeder-plugin/wikicommons-feeder/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# wikicommons-feeder sidecar. Multi-stage build; build context is this repo's root. +# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied +# in — `git` must be present in the builder for cargo to fetch it. + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=wikicommons-feeder-target,target=/build/target \ + cargo build --release --bin wikicommons-feeder \ + && cp target/release/wikicommons-feeder /wikicommons-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /wikicommons-feeder /usr/local/bin/wikicommons-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/wikicommons-feeder"] diff --git a/feeder-plugin/wikicommons-feeder/src/lib.rs b/feeder-plugin/wikicommons-feeder/src/lib.rs new file mode 100644 index 0000000..9bb5d64 --- /dev/null +++ b/feeder-plugin/wikicommons-feeder/src/lib.rs @@ -0,0 +1,4 @@ +//! `wikicommons-feeder` library surface. The binary (`main.rs`) wraps the plugin in +//! [`meta_feeder_sdk::serve_feeders`]. + +pub mod wikicommons; diff --git a/feeder-plugin/wikicommons-feeder/src/main.rs b/feeder-plugin/wikicommons-feeder/src/main.rs new file mode 100644 index 0000000..5e9bb7c --- /dev/null +++ b/feeder-plugin/wikicommons-feeder/src/main.rs @@ -0,0 +1,30 @@ +//! `wikicommons-feeder` — a single-upstream feeder sidecar. +//! +//! Upstream: Wikimedia Commons (commons.wikimedia.org) +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use meta_feeder_sdk::{serve_feeders, FeederPlugin}; +use wikicommons_feeder::wikicommons::WikicommonsPlugin; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(WikicommonsPlugin::new())]; + serve_feeders(plugins, state_dir, listen).await +} diff --git a/feeder-plugin/wikicommons-feeder/src/wikicommons.rs b/feeder-plugin/wikicommons-feeder/src/wikicommons.rs new file mode 100644 index 0000000..f7c8c32 --- /dev/null +++ b/feeder-plugin/wikicommons-feeder/src/wikicommons.rs @@ -0,0 +1,989 @@ +//! Wikimedia Commons bridge via the [MediaWiki Action API](https://commons.wikimedia.org/w/api.php). +//! +//! - Search: `GET /w/api.php?action=query&generator=search&gsrsearch=&gsrnamespace=6&gsrlimit=N&prop=imageinfo&iiprop=url|mime|size|extmetadata&format=json` +//! — one round-trip that returns both search hits and per-file `imageinfo` +//! (direct URL, mime, size, license, artist, description). `srnamespace=6` +//! pins results to the `File:` namespace so we only get media records. +//! - Fetch one: `GET /w/api.php?action=query&titles=File:&prop=imageinfo&...` +//! — single-record metadata lookup used by `compute_outcomes`. +//! - File download: the absolute URL from `imageinfo[0].url` +//! (typically `https://upload.wikimedia.org/wikipedia/commons/<hash>/<file>`). +//! +//! No auth, no API key. Anonymous use is fine for the read paths we use; +//! the API guide asks for a polite `User-Agent` (already supplied). +//! +//! **File-size cap.** Wikimedia Commons hosts files up to several GB +//! (long-form video). The auto-store path buffers the bytes into RAM +//! before writing to local meta-core's WebDAV — gigabytes there would +//! OOM the gateway peer. This plugin refuses anything over +//! [`MAX_FILE_BYTES`] (50 MB) with a permanent error, leaving streaming +//! support for a later feature. +//! +//! See `docs/gateway-feature.md` §4 for the v0 plugin role. + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; + +use async_trait::async_trait; +use serde::Deserialize; + +use meta_feeder_sdk::common; +use meta_feeder_sdk::cache::MidhashCache; +use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, GatewayQuery, HashOutcome}; +use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth}; + +/// Wikimedia Commons API root. Overridable by tests to point at a +/// `wiremock::MockServer`. +const DEFAULT_BASE_URL: &str = "https://commons.wikimedia.org"; + +/// HTTP timeout per upstream call. Generous to cover the occasional +/// upload-server cold-cache fetch for larger media files (image/video). +const HTTP_TIMEOUT_SECS: u64 = 60; + +/// Polite identification — Wikimedia's API policy requires a meaningful +/// User-Agent so they can rate-limit per client. +const USER_AGENT: &str = concat!( + "meta-share/", + env!("CARGO_PKG_VERSION"), + " (gateway:wikicommons)" +); + +/// Hard ceiling on the file size we'll pull through `compute_outcomes`. +/// 50 MB comfortably covers all common-image, most audio, and small +/// video records on Commons; rejects the multi-GB long-form videos +/// that would OOM the gateway peer's buffer-to-RAM auto-store path. +/// Streaming support relaxes this when the auto-store pipeline goes +/// chunked (v0.3 territory). +const MAX_FILE_BYTES: u64 = 50 * 1024 * 1024; + +/// Thumbnail width (in pixels) requested via MediaWiki's `iiurlwidth` +/// parameter. 400 keeps the resulting JPEG/PNG in the ~30–80 KB range +/// per image — small enough to fetch + seed per search result without +/// blowing the search-latency budget, large enough that an inline card +/// preview doesn't look pixelated. +const PREVIEW_WIDTH_PX: u32 = 400; + +/// Wikimedia Commons gateway plugin. Same structural shape as the other +/// fetch-capable plugins. +pub struct WikicommonsPlugin { + http: reqwest::Client, + base_url: String, + cache: Option<MidhashCache>, +} + +impl Default for WikicommonsPlugin { + fn default() -> Self { + Self::new() + } +} + +impl WikicommonsPlugin { + pub fn new() -> Self { + Self::with_base_url(DEFAULT_BASE_URL.to_string()) + } + + /// Override the API root for tests. The mock serves both + /// `/w/api.php` (search + imageinfo) and the upload paths returned + /// in the `imageinfo[0].url` field — the test fixture builds those + /// URLs against `server.uri()` so the same host handles all calls. + pub fn with_base_url(base_url: String) -> Self { + let http = common::build_http_client(HTTP_TIMEOUT_SECS, USER_AGENT, None); + Self { + http, + base_url, + cache: None, + } + } + + fn cache(&self) -> Result<&MidhashCache, GatewayError> { + common::require_cache(self.cache.as_ref(), "wikicommons") + } + + /// Build the `/w/api.php` URL for one of our two query shapes: + /// - search: `extra` = `generator=search&gsrsearch=<q>&gsrnamespace=6&gsrlimit=<n>` + /// - single: `extra` = `titles=File:<title>` + /// + /// The common bits (`action=query&prop=imageinfo&iiprop=...&format=json`) + /// live here so the two callers stay short. + /// + /// `iiurlwidth=<PREVIEW_WIDTH_PX>` asks MediaWiki to also include a + /// thumbnail URL (`thumburl`) in each `imageinfo[0]`. `handle_query` + /// surfaces that as a raw `preview_url` field; the gateway core fetches, + /// seeds, and rewrites it to a `preview` cid (plan §2c). Audio/video + /// records lack a thumbnail (MediaWiki omits `thumburl` for non-image + /// mimes); those ship without a `preview_url`. + fn api_url(&self, extra: &str) -> String { + format!( + "{}/w/api.php?action=query&prop=imageinfo&iiprop=url%7Cmime%7Csize%7Cextmetadata&iiurlwidth={}&format=json&formatversion=1&{}", + self.base_url.trim_end_matches('/'), + PREVIEW_WIDTH_PX, + extra + ) + } + + async fn fetch_api(&self, url: &str) -> Result<MediawikiResponse, GatewayError> { + let resp = self + .http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + common::map_status(&resp)?; + resp.json::<MediawikiResponse>() + .await + .map_err(|e| GatewayError::Permanent(format!("decode mediawiki response: {e}"))) + } + + /// Free-text search on the File: namespace via `generator=search`. + /// Single round-trip returns search hits and their `imageinfo` — no + /// per-record lookup needed at search time. + async fn search_files( + &self, + query: &str, + max_results: usize, + ) -> Result<Vec<MediawikiPage>, GatewayError> { + let url = self.api_url(&format!( + "generator=search&gsrsearch={}&gsrnamespace=6&gsrlimit={}", + common::urlencode(query), + max_results + )); + let body = self.fetch_api(&url).await?; + Ok(body.into_ordered_pages()) + } + + /// Single-record lookup by canonical title. `record_id` is the title + /// **without** the `File:` prefix (that's how we record it). We add + /// the prefix back before the API call. + async fn fetch_page(&self, record_id: &str) -> Result<MediawikiPage, GatewayError> { + let url = self.api_url(&format!("titles=File:{}", common::urlencode(record_id))); + let body = self.fetch_api(&url).await?; + let pages = body.into_ordered_pages(); + let page = pages.into_iter().next().ok_or(GatewayError::NotFound)?; + // MediaWiki returns a synthetic `{"missing": ""}` page for + // unknown titles. Detect by absence of pageid + imageinfo. + if page.imageinfo.as_deref().unwrap_or(&[]).is_empty() && page.pageid.is_none() { + return Err(GatewayError::NotFound); + } + Ok(page) + } + + /// Pull the binary file bytes from the absolute URL Wikimedia returned + /// in `imageinfo[0].url` (full-file fetch for `compute_outcomes`) or + /// `imageinfo[0].thumburl` (thumbnail fetch for `handle_query` → + /// preview seed). The URL points at `upload.wikimedia.org` for real + /// Commons content; the test fixture replaces it with the mock + /// server's URL, so we just trust whatever the API gave us. + async fn fetch_file(&self, url: &str) -> Result<bytes::Bytes, GatewayError> { + common::fetch_bytes(&self.http, url).await + } +} + +#[async_trait] +impl FeederPlugin for WikicommonsPlugin { + fn upstream_id(&self) -> &'static str { + "wikicommons" + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + self.cache = Some(common::open_midhash_cache(cache_dir, "wikicommons")?); + Ok(()) + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result<Vec<DiscoveryRecord>, GatewayError> { + // Layer A early-return: a `fileType:` / `contentKind:` filter + // wholly outside this plugin's served set can't ever match, + // so don't burn an upstream API call. Wikicommons serves + // image/audio/video/document/other and no content kinds, so + // this only rejects a query like `contentKind:movie` or a + // bespoke `fileType:archive` (none of which Commons answers). + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + // Layer A pushdown: if the user filtered by `fileType`, fold + // the requested types into MediaWiki's `filetype:` CirrusSearch + // keyword so the upstream returns the right kind in the first + // place. Without this, broad-spectrum Commons returns jpgs for + // a `fileType:video` query and the dispatcher post-filter has + // to throw most of them away. See `mediawiki_filetype_clause` + // for the meta-vocabulary → CirrusSearch value mapping. + let base_q = query.free_text_or_star(); + let composed_q; + let q = match mediawiki_filetype_clause(query) { + Some(clause) => { + composed_q = format!("{base_q} {clause}"); + composed_q.as_str() + } + None => base_q, + }; + let pages = self.search_files(q, max_results).await?; + // Emit each result's thumbnail URL as a raw `preview_url` field; the + // gateway core fetches it, seeds the bytes into its bitswap blockstore, + // and rewrites it to a content-addressed `preview` cid (plan §2c) — a + // feeder can't reach the core blockstore. Non-image records (audio / + // video / pdf) carry no thumbnail and ship preview-less. + let records: Vec<DiscoveryRecord> = pages + .into_iter() + .filter_map(|p| { + let thumb_url = p + .imageinfo + .as_ref() + .and_then(|v| v.first()) + .and_then(|info| info.thumburl.clone()); + let mut rec = into_discovery_record(p)?; + if let Some(u) = thumb_url { + rec.fields.insert("preview_url".to_string(), u); + } + Some(rec) + }) + .take(max_results) + .collect(); + Ok(records) + } + + async fn compute_outcomes(&self, record_id: &str) -> Result<Vec<HashOutcome>, GatewayError> { + let cache = self.cache()?; + if let Some(hit) = common::cached_outcome(cache, record_id, "wikicommons")? { + return Ok(hit); + } + + let page = self.fetch_page(record_id).await?; + let info = page + .imageinfo + .as_ref() + .and_then(|v| v.first()) + .ok_or_else(|| { + GatewayError::Permanent(format!( + "wikicommons record `{record_id}` has no imageinfo" + )) + })?; + // Refuse oversized files before downloading so a multi-GB video + // doesn't OOM the gateway peer. The 50 MB ceiling covers the + // overwhelming majority of Commons records (images + small + // audio/video); streaming support relaxes this later. + if let Some(size) = info.size { + if size > MAX_FILE_BYTES { + return Err(GatewayError::Permanent(format!( + "wikicommons record `{record_id}` is {size} bytes — exceeds gateway plugin's \ + {MAX_FILE_BYTES}-byte ceiling. Streaming auto-store would lift this; not in v0." + ))); + } + } + let url = &info.url; + let bytes = self.fetch_file(url).await?; + // Defence-in-depth: even if `size` was None / missing in the + // imageinfo, the downloaded body shouldn't exceed the same cap. + if bytes.len() as u64 > MAX_FILE_BYTES { + return Err(GatewayError::Permanent(format!( + "wikicommons record `{record_id}` body was {} bytes — exceeds {MAX_FILE_BYTES} ceiling", + bytes.len() + ))); + } + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + + common::store_midhash(cache, record_id, "wikicommons", &cid); + + let file_extension = file_extension_for(record_id); + let record = into_discovery_record(page).ok_or_else(|| { + // We just successfully fetched a page; this should not + // happen unless imageinfo went missing between API calls. + GatewayError::Permanent(format!( + "wikicommons record `{record_id}` lost imageinfo between calls" + )) + })?; + Ok(common::single_outcome(cid, bytes, record, file_extension)) + } + + fn health(&self) -> PluginHealth { + if self.cache.is_some() { + PluginHealth::Ok + } else { + PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + } + } + } + + fn served_file_types(&self) -> &'static [&'static str] { + // Wikicommons holds heterogeneous mime types — `media_type_for` + // buckets each result into one of meta-core's `fileType` values. + // `other` covers everything we can't classify. + &["image", "audio", "video", "document", "other"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + // No semantic refinement yet — wikicommons records carry only + // the coarse mime bucket. Future: photo / illustration / gif + // classification could land here. + &[] + } +} + +// -- MediaWiki JSON shapes (subset we use) ---------------------------------- + +#[derive(Debug, Default, Deserialize)] +struct MediawikiResponse { + #[serde(default)] + query: Option<MediawikiQuery>, +} + +impl MediawikiResponse { + /// Pages come back as a `pages` map keyed by stringified pageid. + /// `generator=search` adds an `index` field to each page reflecting + /// the original search ranking — we sort on that so the caller sees + /// hits in relevance order. Single-record lookups have no `index` + /// and reach `into_ordered_pages` with at most one entry anyway. + fn into_ordered_pages(self) -> Vec<MediawikiPage> { + let Some(query) = self.query else { + return Vec::new(); + }; + let mut pages: Vec<MediawikiPage> = query.pages.into_values().collect(); + pages.sort_by_key(|p| p.index.unwrap_or(u32::MAX)); + pages + } +} + +#[derive(Debug, Default, Deserialize)] +struct MediawikiQuery { + #[serde(default)] + pages: HashMap<String, MediawikiPage>, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct MediawikiPage { + pageid: Option<u64>, + title: Option<String>, + /// Search-rank position (1-based) — only populated when the page + /// came from `generator=search`. Lower = more relevant; we sort on + /// this so search results don't get scrambled by `HashMap` order. + index: Option<u32>, + imageinfo: Option<Vec<MediawikiImageInfo>>, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct MediawikiImageInfo { + url: String, + #[serde(default)] + mime: Option<String>, + #[serde(default)] + size: Option<u64>, + /// Scaled-down preview URL — only populated when the request + /// included `iiurlwidth=<n>` AND the file is an image. For + /// audio/video records MediaWiki omits this field even when + /// `iiurlwidth` was set (no thumbnail exists), and we ship the + /// record without a `preview` cid. + #[serde(default)] + thumburl: Option<String>, + /// Wikimedia's extmetadata block — license, artist, description, + /// date. Each entry is `{ "value": "…", "source": "…" }`; we keep + /// only the value via the inner struct. + #[serde(default)] + extmetadata: Option<MediawikiExtMetadata>, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default, rename_all = "PascalCase")] +struct MediawikiExtMetadata { + artist: Option<ExtValue>, + license_short_name: Option<ExtValue>, + image_description: Option<ExtValue>, + date_time_original: Option<ExtValue>, + credit: Option<ExtValue>, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct ExtValue { + value: String, +} + +/// Map a MediaWiki `page` (with imageinfo) into a `DiscoveryRecord`. +/// Returns `None` when the page has no imageinfo entry — happens for +/// deleted files where the page still exists, and for the synthetic +/// `missing=""` page MediaWiki emits for unknown titles. +fn into_discovery_record(page: MediawikiPage) -> Option<DiscoveryRecord> { + let title = page.title?; + // `record_id` is the title with the `File:` prefix stripped — that's + // how the rest of the gateway tier refers to it (URL paths, library + // entries, the synthetic `gateway:wikicommons:<id>` CID). + let record_id = title + .strip_prefix("File:") + .map(str::to_string) + .unwrap_or(title.clone()); + let info = page.imageinfo?.into_iter().next()?; + let mut fields: BTreeMap<String, String> = BTreeMap::new(); + fields.insert("title".to_string(), record_id.clone()); + fields.insert("fileType".to_string(), media_type_for(info.mime.as_deref())); + fields.insert( + "sourceUrl".to_string(), + format!( + "https://commons.wikimedia.org/wiki/{}", + encode_title_for_url(&title) + ), + ); + fields.insert("fileName".to_string(), record_id.clone()); + fields.insert(upstream_id_field("wikicommons"), record_id.clone()); + // The `preview` field (a content-addressed cid for an inline + // thumbnail) is NOT set here — `handle_query` seeds the bytes into + // the gateway's blockstore in parallel and attaches the cid + // post-hoc. We deliberately don't emit an upstream URL field; the + // design is "every byte reference is a cid", so dashboards never + // need to know what `upload.wikimedia.org` is. + if let Some(mime) = info.mime.as_deref() { + fields.insert("mime".to_string(), mime.to_string()); + } + if let Some(ext) = file_extension_for(&record_id) { + fields.insert("format".to_string(), ext); + } + if let Some(size) = info.size { + fields.insert("size".to_string(), size.to_string()); + } + if let Some(em) = info.extmetadata { + if let Some(v) = em.artist { + fields.insert("artist".to_string(), v.value); + } + if let Some(v) = em.license_short_name { + fields.insert("license".to_string(), v.value); + } + if let Some(v) = em.image_description { + fields.insert("description".to_string(), v.value); + } + if let Some(v) = em.date_time_original { + fields.insert("dateOriginal".to_string(), v.value); + } + if let Some(v) = em.credit { + fields.insert("credit".to_string(), v.value); + } + } + Some(DiscoveryRecord { + upstream_id: "wikicommons".to_string(), + record_id, + fields, + }) +} + +/// Bucket a MediaWiki mime into a meta-core `fileType` value +/// (vocabulary defined in +/// [docs/project-architecture/metadata-keys.md → `fileType`]). +/// `image/jpeg` → "image", `audio/ogg` → "audio", `video/webm` → "video", +/// `application/pdf` → "document", everything else → "other". +fn media_type_for(mime: Option<&str>) -> String { + let Some(mime) = mime else { + return "other".to_string(); + }; + if mime.starts_with("image/") { + "image".to_string() + } else if mime.starts_with("audio/") { + "audio".to_string() + } else if mime.starts_with("video/") { + "video".to_string() + } else if mime.starts_with("application/pdf") { + "document".to_string() + } else { + "other".to_string() + } +} + +/// File extension from the canonical title. Mirror of arxiv / pubmed's +/// per-plugin hardcoded `"pdf"`, but Commons records carry heterogeneous +/// types — derive from the title (which Wikimedia normalises to include +/// the extension). Returns the lowercased extension without the dot, or +/// `None` if the title has no recognisable extension. +fn file_extension_for(record_id: &str) -> Option<String> { + let dot = record_id.rfind('.')?; + let ext = &record_id[dot + 1..]; + if ext.is_empty() || ext.len() > 5 || !ext.chars().all(|c| c.is_ascii_alphanumeric()) { + return None; + } + Some(ext.to_ascii_lowercase()) +} + +/// Percent-encode a MediaWiki title for the wiki page URL. +/// Wikimedia uses underscores for spaces in URLs (the API normalises +/// spaces → underscores anyway), and percent-encodes the rest. We keep +/// it minimal: replace spaces, percent-encode everything else outside +/// the unreserved set. +fn encode_title_for_url(title: &str) -> String { + let with_underscores: String = title + .chars() + .map(|c| if c == ' ' { '_' } else { c }) + .collect(); + let mut out = String::with_capacity(with_underscores.len()); + for c in with_underscores.chars() { + match c { + 'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' | '/' | ':' => out.push(c), + _ => { + let mut buf = [0u8; 4]; + for b in c.encode_utf8(&mut buf).as_bytes() { + out.push_str(&format!("%{:02X}", b)); + } + } + } + } + out +} + +/// Percent-encode a value for the query string. Same safe-set the other +/// plugins use; covers MediaWiki's search query syntax. +/// Map `query.filters["fileType"]` (our cohort vocabulary) into a +/// MediaWiki CirrusSearch `filetype:` clause that Commons' search +/// understands. Returns `None` when the query has no `fileType:` +/// filter, so the caller can leave the upstream query unmodified. +/// +/// MediaWiki vocabulary (`filetype:` keyword on Commons): `bitmap`, +/// `drawing`, `audio`, `video`, `office`, `unknown`. Mapping from our +/// coarse `fileType` enum: +/// +/// | meta-cohort `fileType` | MediaWiki `filetype:` value(s) | +/// |---|---| +/// | `image` | `bitmap`, `drawing` | +/// | `audio` | `audio` | +/// | `video` | `video` | +/// | `document` | `office` | +/// | (other) | dropped — Commons has no concept | +/// +/// CirrusSearch syntax for OR-ing filetype values is **pipe-separated +/// inside a single keyword**: `filetype:bitmap|drawing`. Using +/// explicit `OR` between two `filetype:` keywords parses as an +/// impossible AND (a file can't be both `bitmap` AND `drawing`), so +/// emit the pipe form unconditionally. +fn mediawiki_filetype_clause(query: &GatewayQuery) -> Option<String> { + let requested = query.filters.get("fileType")?; + if requested.is_empty() { + return None; + } + let mut mapped: Vec<&'static str> = Vec::new(); + for v in requested { + let extra: &[&'static str] = match v.to_ascii_lowercase().as_str() { + "image" => &["bitmap", "drawing"], + "audio" => &["audio"], + "video" => &["video"], + "document" => &["office"], + // `other`, `subtitle`, `archive`, … — no Commons equivalent. + // Skip rather than emit a syntactically-broken clause; the + // dispatcher's post-filter (Layer B) will drop anything + // that comes back with the wrong fileType anyway. + _ => &[], + }; + for e in extra { + if !mapped.iter().any(|m| m == e) { + mapped.push(e); + } + } + } + if mapped.is_empty() { + return None; + } + Some(format!("filetype:{}", mapped.join("|"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Sample `generator=search` response with two image hits. + /// Real Wikimedia API output, trimmed. + const SAMPLE_SEARCH: &str = r#"{ + "batchcomplete": "", + "query": { + "pages": { + "1": { + "pageid": 1, + "ns": 6, + "title": "File:Sunrise over the sea.jpg", + "index": 1, + "imageinfo": [ + { + "url": "https://upload.wikimedia.org/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg", + "mime": "image/jpeg", + "size": 234567, + "extmetadata": { + "Artist": { "value": "Jane Doe", "source": "commons" }, + "LicenseShortName": { "value": "CC BY-SA 4.0", "source": "commons" }, + "ImageDescription": { "value": "A sunrise photographed off the coast.", "source": "commons" }, + "DateTimeOriginal": { "value": "2023-08-12 06:14:00", "source": "exif" }, + "Credit": { "value": "Own work", "source": "commons" } + } + } + ] + }, + "2": { + "pageid": 2, + "ns": 6, + "title": "File:Echinacea_purpurea.jpg", + "index": 2, + "imageinfo": [ + { + "url": "https://upload.wikimedia.org/wikipedia/commons/x/yz/Echinacea_purpurea.jpg", + "mime": "image/jpeg", + "size": 145000 + } + ] + } + } + } + }"#; + + fn empty_gateway_query() -> GatewayQuery { + GatewayQuery { + raw_text: String::new(), + free_text: String::new(), + filters: std::collections::BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + } + } + + #[test] + fn mediawiki_filetype_clause_returns_none_for_unfiltered_query() { + assert_eq!(mediawiki_filetype_clause(&empty_gateway_query()), None); + } + + #[test] + fn mediawiki_filetype_clause_maps_video() { + let mut q = empty_gateway_query(); + q.filters.insert("fileType".into(), vec!["video".into()]); + assert_eq!( + mediawiki_filetype_clause(&q), + Some("filetype:video".to_string()) + ); + } + + #[test] + fn mediawiki_filetype_clause_maps_image_to_bitmap_and_drawing() { + let mut q = empty_gateway_query(); + q.filters.insert("fileType".into(), vec!["image".into()]); + assert_eq!( + mediawiki_filetype_clause(&q), + Some("filetype:bitmap|drawing".to_string()) + ); + } + + #[test] + fn mediawiki_filetype_clause_ors_multiple_requested_types() { + let mut q = empty_gateway_query(); + q.filters + .insert("fileType".into(), vec!["video".into(), "audio".into()]); + assert_eq!( + mediawiki_filetype_clause(&q), + Some("filetype:video|audio".to_string()) + ); + } + + #[test] + fn mediawiki_filetype_clause_dedupes_overlapping_mappings() { + let mut q = empty_gateway_query(); + // image → [bitmap, drawing]; document → [office]; both kept. + q.filters.insert( + "fileType".into(), + vec!["image".into(), "document".into(), "image".into()], + ); + assert_eq!( + mediawiki_filetype_clause(&q), + Some("filetype:bitmap|drawing|office".to_string()) + ); + } + + #[test] + fn mediawiki_filetype_clause_drops_unmappable_values() { + let mut q = empty_gateway_query(); + q.filters + .insert("fileType".into(), vec!["archive".into(), "subtitle".into()]); + assert_eq!(mediawiki_filetype_clause(&q), None); + } + + #[test] + fn parses_search_response_in_index_order() { + let resp: MediawikiResponse = serde_json::from_str(SAMPLE_SEARCH).expect("parse"); + let pages = resp.into_ordered_pages(); + assert_eq!(pages.len(), 2); + assert_eq!( + pages[0].title.as_deref(), + Some("File:Sunrise over the sea.jpg") + ); + assert_eq!( + pages[1].title.as_deref(), + Some("File:Echinacea_purpurea.jpg") + ); + } + + #[test] + fn into_discovery_record_emits_required_fields() { + let resp: MediawikiResponse = serde_json::from_str(SAMPLE_SEARCH).unwrap(); + let pages = resp.into_ordered_pages(); + let rec = into_discovery_record(pages.into_iter().next().unwrap()).expect("rec"); + assert_eq!(rec.upstream_id, "wikicommons"); + assert_eq!(rec.record_id, "Sunrise over the sea.jpg"); + // §6.6 conventions: fileType (mime bucket) + no contentKind (no + // semantic refinement on wikicommons records yet). + assert_eq!( + rec.fields.get("fileType").map(String::as_str), + Some("image") + ); + assert!(!rec.fields.contains_key("contentKind")); + assert_eq!(rec.fields.get("format").map(String::as_str), Some("jpg")); + assert_eq!( + rec.fields.get("sourceUrl").map(String::as_str), + Some("https://commons.wikimedia.org/wiki/File:Sunrise_over_the_sea.jpg") + ); + assert_eq!( + rec.fields.get("fileName").map(String::as_str), + Some("Sunrise over the sea.jpg") + ); + // Canonical `<upstream_id>id` field. + assert_eq!( + rec.fields.get("wikicommonsid").map(String::as_str), + Some("Sunrise over the sea.jpg") + ); + // Extmetadata surface. + assert_eq!( + rec.fields.get("artist").map(String::as_str), + Some("Jane Doe") + ); + assert_eq!( + rec.fields.get("license").map(String::as_str), + Some("CC BY-SA 4.0") + ); + assert_eq!( + rec.fields.get("description").map(String::as_str), + Some("A sunrise photographed off the coast.") + ); + } + + #[test] + fn media_type_buckets_match_mime() { + assert_eq!(media_type_for(Some("image/jpeg")), "image"); + assert_eq!(media_type_for(Some("audio/ogg")), "audio"); + assert_eq!(media_type_for(Some("video/webm")), "video"); + assert_eq!(media_type_for(Some("application/pdf")), "document"); + assert_eq!(media_type_for(Some("application/octet-stream")), "other"); + assert_eq!(media_type_for(None), "other"); + } + + #[test] + fn file_extension_for_handles_common_shapes() { + assert_eq!(file_extension_for("Sunrise.jpg").as_deref(), Some("jpg")); + assert_eq!(file_extension_for("Foo.OGG").as_deref(), Some("ogg")); + assert_eq!(file_extension_for("Foo.webm").as_deref(), Some("webm")); + // Too long to be an extension. + assert!(file_extension_for("Foo.thisisalonglongtail").is_none()); + // No extension at all. + assert!(file_extension_for("BareName").is_none()); + // Non-alphanumeric trailing — refused. + assert!(file_extension_for("Foo.tar.gz?x").is_none()); + } + + #[test] + fn encode_title_swaps_spaces_for_underscores() { + assert_eq!( + encode_title_for_url("File:Sunrise over the sea.jpg"), + "File:Sunrise_over_the_sea.jpg" + ); + assert_eq!( + encode_title_for_url("File:Caf\u{00e9}.jpg"), + "File:Caf%C3%A9.jpg" + ); + } + + // -- HTTP integration tests against a wiremock server ---------------- + + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn configured_plugin_against(server: &MockServer) -> (WikicommonsPlugin, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let mut plugin = WikicommonsPlugin::with_base_url(server.uri()); + plugin.configure(dir.path()).expect("configure"); + (plugin, dir) + } + + #[tokio::test] + async fn handle_query_maps_search_to_discovery_records() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/w/api.php")) + .and(query_param("generator", "search")) + .and(query_param("gsrsearch", "sunrise")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(SAMPLE_SEARCH) + .insert_header("content-type", "application/json"), + ) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("sunrise"), 10) + .await + .expect("handle_query"); + assert_eq!(records.len(), 2); + assert_eq!(records[0].record_id, "Sunrise over the sea.jpg"); + } + + #[tokio::test] + async fn handle_query_truncates_to_max_results() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/w/api.php")) + .respond_with(ResponseTemplate::new(200).set_body_string(SAMPLE_SEARCH)) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("sunrise"), 1) + .await + .expect("handle_query"); + assert_eq!(records.len(), 1); + } + + #[tokio::test] + async fn compute_outcomes_fetches_file_and_caches() { + let server = MockServer::start().await; + // Build a single-record response that points at the mock's own + // host, so the file fetch is intercepted too. + let img_bytes = b"\xFF\xD8\xFF mock jpeg body for hashing\n".to_vec(); + let expected_cid = meta_feeder_sdk::hash::compute_ipfs_cid(&img_bytes); + let single = format!( + r#"{{ + "batchcomplete": "", + "query": {{ + "pages": {{ + "1": {{ + "pageid": 1, + "ns": 6, + "title": "File:Sunrise over the sea.jpg", + "imageinfo": [ + {{ + "url": "{base}/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg", + "mime": "image/jpeg", + "size": {sz} + }} + ] + }} + }} + }} + }}"#, + base = server.uri(), + sz = img_bytes.len() + ); + Mock::given(method("GET")) + .and(path("/w/api.php")) + .and(query_param("titles", "File:Sunrise over the sea.jpg")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(single) + .insert_header("content-type", "application/json"), + ) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/wikipedia/commons/a/ab/Sunrise_over_the_sea.jpg")) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(img_bytes.clone()) + .insert_header("content-type", "image/jpeg"), + ) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let outcome = plugin + .compute_outcomes("Sunrise over the sea.jpg") + .await + .expect("compute") + .into_iter() + .next() + .expect("single outcome"); + assert_eq!(outcome.hash.as_str(), expected_cid); + assert_eq!(outcome.bytes.as_deref(), Some(img_bytes.as_slice())); + assert_eq!(outcome.file_extension.as_deref(), Some("jpg")); + let rec = outcome.record.expect("record present on fresh compute"); + assert_eq!(rec.record_id, "Sunrise over the sea.jpg"); + assert_eq!( + rec.fields.get("fileType").map(String::as_str), + Some("image") + ); + } + + #[tokio::test] + async fn compute_outcomes_cache_hit_skips_http() { + let server = MockServer::start().await; + let (plugin, _dir) = configured_plugin_against(&server); + plugin + .cache + .as_ref() + .unwrap() + .put_midhash("Sunrise.jpg", "bafyCACHED") + .unwrap(); + let outcome = plugin + .compute_outcomes("Sunrise.jpg") + .await + .expect("cache hit") + .into_iter() + .next() + .expect("single outcome"); + assert_eq!(outcome.hash.as_str(), "bafyCACHED"); + assert!(outcome.bytes.is_none()); + assert!(outcome.record.is_none()); + } + + #[tokio::test] + async fn compute_outcomes_refuses_oversized_records() { + let server = MockServer::start().await; + let too_big = MAX_FILE_BYTES + 1; + let single = format!( + r#"{{ + "batchcomplete": "", + "query": {{ + "pages": {{ + "1": {{ + "pageid": 1, + "ns": 6, + "title": "File:Huge.webm", + "imageinfo": [ + {{ "url": "{base}/foo.webm", "mime": "video/webm", "size": {sz} }} + ] + }} + }} + }} + }}"#, + base = server.uri(), + sz = too_big + ); + Mock::given(method("GET")) + .and(path("/w/api.php")) + .respond_with(ResponseTemplate::new(200).set_body_string(single)) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .compute_outcomes("Huge.webm") + .await + .expect_err("oversized"); + assert!(matches!(err, GatewayError::Permanent(_)), "got {err:?}"); + } + + #[tokio::test] + async fn upstream_404_maps_to_not_found() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/w/api.php")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .compute_outcomes("Does-not-exist.jpg") + .await + .expect_err("expected NotFound"); + assert!(matches!(err, GatewayError::NotFound), "got {err:?}"); + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]