From f42b92855c2977dd3fda4db9c2c02a7da8d4ba4a Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 24 Aug 2026 19:08:25 +0200 Subject: [PATCH] feeder rework --- .gitattributes | 6 + .github/workflows/publish.yml | 59 + Cargo.lock | 1948 +++++++++++++++++ Cargo.toml | 9 + README.md | 18 +- .../internetarchive-feeder/Cargo.toml | 42 + .../internetarchive-feeder/Dockerfile | 44 + .../internetarchive-feeder/src/consts.rs | 87 + .../src/internetarchive.rs | 1311 +++++++++++ .../internetarchive-feeder/src/lib.rs | 6 + .../internetarchive-feeder/src/main.rs | 33 + rust-toolchain.toml | 3 + 12 files changed, 3565 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/publish.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 feeder-plugin/internetarchive-feeder/Cargo.toml create mode 100644 feeder-plugin/internetarchive-feeder/Dockerfile create mode 100644 feeder-plugin/internetarchive-feeder/src/consts.rs create mode 100644 feeder-plugin/internetarchive-feeder/src/internetarchive.rs create mode 100644 feeder-plugin/internetarchive-feeder/src/lib.rs create mode 100644 feeder-plugin/internetarchive-feeder/src/main.rs create mode 100644 rust-toolchain.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..de7270a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/paper-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/internetarchive-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..104961e --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1948 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "internetarchive-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "meta-feeder-sdk", + "redb", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..d721f8a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-internetarchive" diff --git a/README.md b/README.md index 10f54b2..1fe2893 100644 --- a/README.md +++ b/README.md @@ -1 +1,17 @@ -# meta-feeder-internetarchive \ No newline at end of file +# meta-feeder-internetarchive + +Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless. + +**Upstream:** Internet Archive (archive.org) +**Role:** bytes + +A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin` +from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk). + +```bash +cargo build --release --bin internetarchive-feeder +cargo test +``` + +Scope and naming conventions: `docs/project-architecture/feeder-architecture.md` +in the meta-root. diff --git a/feeder-plugin/internetarchive-feeder/Cargo.toml b/feeder-plugin/internetarchive-feeder/Cargo.toml new file mode 100644 index 0000000..1ffbd2c --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "internetarchive-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless. +# Upstream: Internet Archive (archive.org) + +[lib] +name = "internetarchive_feeder" +path = "src/lib.rs" + +[[bin]] +name = "internetarchive-feeder" +path = "src/main.rs" + +[dependencies] +# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to: +# meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "" } +meta-feeder-sdk = { path = "../../../meta-feeder-sdk" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +bytes = "1" +futures = "0.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +redb = "2" +base64 = "0.22" + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/internetarchive-feeder/Dockerfile b/feeder-plugin/internetarchive-feeder/Dockerfile new file mode 100644 index 0000000..efb2b42 --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# internetarchive-feeder sidecar. Multi-stage build; build context is this repo's root. +# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied +# in — `git` must be present in the builder for cargo to fetch it. + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=internetarchive-feeder-target,target=/build/target \ + cargo build --release --bin internetarchive-feeder \ + && cp target/release/internetarchive-feeder /internetarchive-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /internetarchive-feeder /usr/local/bin/internetarchive-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/internetarchive-feeder"] diff --git a/feeder-plugin/internetarchive-feeder/src/consts.rs b/feeder-plugin/internetarchive-feeder/src/consts.rs new file mode 100644 index 0000000..b7b1017 --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/src/consts.rs @@ -0,0 +1,87 @@ +//! Constants for `internetarchive-feeder`. + +/// User-Agent for outbound HTTP. +/// +/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent +/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to +/// misdiagnose. Identify the application and carry contact information. +pub const USER_AGENT: &str = concat!( + "internetarchive-feeder/", + env!("CARGO_PKG_VERSION"), + " ( https://github.com/worph/meta-feeder-internetarchive )" +); + +/// Internet Archive advanced-search endpoint. Keyless. +pub const IA_SEARCH_URL: &str = "https://archive.org/advancedsearch.php"; +/// Internet Archive per-item file-listing endpoint (`/metadata/`). +pub const IA_METADATA_BASE: &str = "https://archive.org/metadata"; +/// Internet Archive file-download base (`/download//`). +pub const IA_DOWNLOAD_BASE: &str = "https://archive.org/download"; +/// The archive's own thumbnail service, keyed by item identifier. +/// +/// ⚠ **Without this the entire free tier is invisible.** The consumer's quality +/// gate requires a cover, and an archive item's artwork is an ordinary file in +/// its file list under an unpredictable name — so a first cut emitted no +/// artwork field at all and every free-tier track landed in the raw bucket. The +/// cold-start row, whose whole purpose is to be full on a client with nothing +/// configured, was permanently empty. +/// +/// This endpoint always answers for a real item (verified: 200 image/png), so +/// it is the one predictable cover source the archive has. +pub const IA_THUMB_BASE: &str = "https://archive.org/services/img"; +/// Sustained Internet Archive request rate. They publish no hard number and +/// ask for reasonable behaviour; this is deliberately modest because a single +/// item listing can be large. +pub const IA_RATE_PER_SEC: f64 = 3.0; +/// Internet Archive burst ceiling. +pub const IA_BURST: f64 = 6.0; +/// Request timeout for an Internet Archive metadata/search call (seconds). +pub const IA_TIMEOUT_SECS: u64 = 30; +/// Timeout for downloading one audio file's bytes (seconds). Generous: a +/// lossless live-set track can be 100 MB. +pub const IA_FETCH_TIMEOUT_SECS: u64 = 600; +/// Hard ceiling on the bytes this feeder will pull for a single track. +/// +/// The gateway core holds the whole outcome in memory before seeding it, so an +/// unbounded fetch is an OOM waiting to happen on a 128 MB feeder. A 200 MB +/// ceiling clears any normal lossless track (a 24/96 FLAC of a 10-minute piece +/// is ~200 MB at the extreme) while refusing a whole-concert single file. +pub const IA_MAX_TRACK_BYTES: u64 = 200 * 1024 * 1024; +/// The Internet Archive collections this feeder searches. +/// +/// Deliberately an allow-list rather than "all of `mediatype:audio`". The +/// archive holds a great deal of audio whose redistribution status is unclear +/// (uploads of commercial recordings, radio airchecks). These four are the +/// collections whose licensing is either explicit per item or structurally +/// clean, which is what makes the `licence` field on the emitted record +/// meaningful rather than decorative: +/// +/// - `etree` — the Live Music Archive: taper recordings of bands who have +/// explicitly permitted taping and distribution. +/// - `netlabels` — net-label releases, uniformly Creative Commons. +/// - `audio_music` — the general music collection; per-item `licenseurl` is +/// what gates an individual record (see `internetarchive.rs`). +/// - `78rpm` — digitised 78s, overwhelmingly public domain by age. +pub const IA_COLLECTIONS: &[&str] = &["etree", "netlabels", "audio_music", "78rpm"]; +/// Archive `mediatype` values that hold music. +/// +/// ⚠ **`etree` is a mediatype, not only a collection**, and missing that is the +/// easiest way to silently lose most of the free tier. Live Music Archive items +/// are `mediatype:etree`; a query restricted to `mediatype:(audio)` matches +/// **zero** of them. Measured on the live endpoint: +/// +/// | query | numFound | +/// |---|---| +/// | `mediatype:(audio) AND collection:(etree)` | **0** | +/// | `mediatype:(etree)` | 293,738 | +/// | `collection:(etree) AND creator:("Grateful Dead")` | 18,334 | +/// +/// The failure is quiet — an `audio`-only query still returns a handful of +/// `audio_music` items, so the search looks like it works and merely appears to +/// have a thin catalogue. +pub const IA_MEDIATYPES: &[&str] = &["audio", "etree"]; +/// Audio file extensions the Internet Archive tier will emit as tracks, +/// preferred order (best quality first). The archive stores several +/// derivatives of the same recording; emitting all of them would publish four +/// records for one track. +pub const IA_AUDIO_EXTS: &[&str] = &["flac", "ogg", "opus", "m4a", "mp3"]; diff --git a/feeder-plugin/internetarchive-feeder/src/internetarchive.rs b/feeder-plugin/internetarchive-feeder/src/internetarchive.rs new file mode 100644 index 0000000..827e52e --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/src/internetarchive.rs @@ -0,0 +1,1311 @@ +//! The `internetarchive` upstream — the **free content tier**. +//! +//! This is the plugin that makes a cold meta-listen client show music it can +//! actually play. Everything else in this crate resolves *identity*; this one +//! resolves *bytes*, with no credential, from collections whose redistribution +//! status is either explicit per item or structurally clean. +//! +//! That is not a convenience. Every other MetaMesh consumer's cold-start state +//! is an empty wall, which is indistinguishable to a user from a broken swarm +//! — a confusion that has cost real debugging time on this stack. A tier that +//! works with nothing configured is the only way to avoid it without weakening +//! the untyped-query load gate. +//! +//! ## Licensing is a field, not a vibe +//! +//! The gateway's auto-store makes the peer a real **host** of whatever this +//! plugin resolves: bytes go to meta-core over WebDAV and get seeded to +//! bitswap. So "may this be redistributed?" has to be answered per record, and +//! the answer is written to `licence` (`METADATA_KEYS.md` §5). +//! +//! Two rules follow, and both are enforced in [`InternetArchivePlugin`]: +//! +//! 1. **Collection allow-list.** `consts::IA_COLLECTIONS`, not all of +//! `mediatype:audio`. The archive holds a great deal of audio whose status +//! is unclear (uploads of commercial recordings, radio airchecks). +//! 2. **A per-item licence, or the item's collection vouches for it.** An item +//! from `etree` (taper recordings of bands who permit taping) or `78rpm` +//! (public domain by age) is clean by collection; anything from the general +//! `audio_music` pool needs its own `licenseurl` or it is skipped. + +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use base64::Engine; +use meta_feeder_sdk::cache::MidhashCache; +use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, GatewayQuery, HashKind, HashOutcome}; +use meta_feeder_sdk::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; +use serde_json::Value; +use tracing::{debug, warn}; + +use crate::consts::{ + IA_AUDIO_EXTS, IA_BURST, IA_COLLECTIONS, IA_DOWNLOAD_BASE, IA_FETCH_TIMEOUT_SECS, IA_MEDIATYPES, + IA_MAX_TRACK_BYTES, IA_METADATA_BASE, IA_RATE_PER_SEC, IA_SEARCH_URL, IA_THUMB_BASE, + IA_TIMEOUT_SECS, + USER_AGENT, +}; +use meta_feeder_sdk::budget::{Lease, RateBudget}; + +/// Collections whose licensing is clean by construction, and the `licence` +/// value an item in them carries when it declares no `licenseurl` of its own. +/// +/// **This table is the whole of rule 2.** A collection absent from it — +/// notably the general `audio_music` pool — vouches for nothing, so an item +/// there needs its own licence URL or it is skipped entirely. Adding a row +/// here is asserting, on behalf of every peer that will end up hosting those +/// bytes, that the collection's contents may be redistributed. +const SELF_VOUCHING_COLLECTIONS: &[(&str, &str)] = &[ + // Taper recordings distributed with the artist's permission. Not a + // public-domain claim and not a CC grant — the archive's own term for it is + // the honest label. + ("etree", "LiveMusicArchive-TapingPolicy"), + // Net-label releases are uniformly Creative Commons, but the exact variant + // is per release, so only the family is asserted. + ("netlabels", "CC"), + // Digitised 78s: public domain by age. + ("78rpm", "PublicDomain"), +]; + +/// The `licence` value for an item from a self-vouching collection. +fn collection_licence(collections: &[String]) -> Option<&'static str> { + collections.iter().find_map(|c| { + SELF_VOUCHING_COLLECTIONS + .iter() + .find(|(name, _)| *name == c.as_str()) + .map(|(_, licence)| *licence) + }) +} + +/// Map a Creative Commons (or other) licence URL to a short identifier. +/// +/// `METADATA_KEYS.md` requires an SPDX-style identifier rather than a URL, +/// because a consumer comparing licences should not be doing string surgery on +/// `http://` vs `https://` and a trailing slash. +pub fn licence_from_url(url: &str) -> Option { + let u = url.trim().trim_end_matches('/').to_ascii_lowercase(); + if u.is_empty() { + return None; + } + if u.contains("publicdomain/zero") { + return Some("CC0-1.0".to_string()); + } + if u.contains("publicdomain/mark") { + return Some("PublicDomain".to_string()); + } + // .../licenses//[/] + let rest = u.split("/licenses/").nth(1)?; + let mut parts = rest.split('/'); + let variant = parts.next()?.to_ascii_uppercase(); + let version = parts.next().unwrap_or("4.0"); + if variant.is_empty() { + return None; + } + Some(format!("CC-{variant}-{version}")) +} + +/// Encode a record id for the single-path-segment `/fetch/:upstream/:record_id` +/// route. +/// +/// ⚠ A file name inside an archive item may contain `/` (items have +/// subdirectories) and axum's `:record_id` matches exactly one segment, so a +/// raw `identifier/filename` would 404 on fetch while working fine through +/// `/compute` (which takes the id in a JSON body). Base64url keeps one shape +/// on both paths. `~` is unreserved in RFC 3986 and cannot appear in an +/// archive identifier, so it is an unambiguous separator. +pub fn track_record_id(identifier: &str, file_name: &str) -> String { + let b64 = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(file_name.as_bytes()); + format!("{identifier}~{b64}") +} + +/// Inverse of [`track_record_id`]. `None` for a bare item id (a pack). +pub fn split_track_record_id(record_id: &str) -> Option<(&str, String)> { + let (identifier, b64) = record_id.split_once('~')?; + let bytes = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(b64.as_bytes()) + .ok()?; + Some((identifier, String::from_utf8(bytes).ok()?)) +} + +/// Rank an audio file name by [`IA_AUDIO_EXTS`] preference. `None` when the +/// extension is not one we serve. +fn audio_rank(name: &str) -> Option { + let ext = name.rsplit_once('.')?.1.to_ascii_lowercase(); + IA_AUDIO_EXTS.iter().position(|e| *e == ext) +} + +/// The recording a derivative file belongs to. +/// +/// An archive item stores several encodings of the same recording, named by +/// appending a derivative marker to the stem: `Track01.flac`, `Track01.mp3`, +/// `Track01_64kb.mp3`, `Track01_vbr.mp3`. Grouping on the bare stem would keep +/// the marker and leave each derivative in its own group, which is how the +/// first cut of this shipped four records for one track. +/// +/// The markers stripped are the ones the archive's own derivation pipeline +/// emits. A title genuinely ending in one of them collapses with a sibling +/// that does not — an acceptable trade for not showing every track four times. +fn recording_stem(name: &str) -> String { + let base = name.rsplit_once('.').map(|(s, _)| s).unwrap_or(name); + let lower = base.to_ascii_lowercase(); + for marker in ["_vbr", "_sample", "_alt"] { + if let Some(s) = lower.strip_suffix(marker) { + return s.trim_end_matches('_').to_string(); + } + } + // `_kb` — the bitrate derivatives. + if let Some(rest) = lower.strip_suffix("kb") { + let trimmed = rest.trim_end_matches(|c: char| c.is_ascii_digit()); + if trimmed.len() < rest.len() && trimmed.ends_with('_') { + return trimmed.trim_end_matches('_').to_string(); + } + } + lower +} + +/// Decode an archive `track` value into `(disc, track)`. +/// +/// The archive's `track` field carries three shapes, all seen live: +/// +/// - `"7"` — a plain ordinal. +/// - `"7/12"` — ordinal out of total. +/// - `"101"` — **disc-encoded**: hundreds digit is the disc, remainder is the +/// track. Standard on Live Music Archive items (`gd85-12-31 s1t01 …` is +/// `track: 101`), where a show routinely spans three or four discs. +/// +/// ⚠ Without the third case a two-disc show reports track numbers 101…115 and +/// 201…212, which sorts correctly by accident and then renders as "track 101" +/// in every UI. A music track number above 99 does not otherwise occur, so the +/// rule is safe. +pub fn decode_track_number(raw: &str) -> Option<(Option, u32)> { + let head = raw.split('/').next()?.trim(); + let n: u32 = head.parse().ok()?; + if n == 0 { + return None; + } + if n >= 100 { + let disc = n / 100; + let track = n % 100; + if track > 0 { + return Some((Some(disc), track)); + } + } + Some((None, n)) +} + +/// One resolved audio file within an item. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArchiveTrack { + pub file_name: String, + pub title: String, + pub track_number: Option, + pub disc_number: Option, + pub size: Option, +} + +/// One archive item, with the audio files worth emitting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArchiveItem { + pub identifier: String, + pub title: String, + pub creator: String, + pub date: Option, + pub licence: Option, + pub tracks: Vec, +} + +/// Pick the best derivative per recording. +/// +/// An archive item stores several encodings of the same recording +/// (`Track.flac`, `Track.mp3`, `Track_64kb.mp3`). Emitting all of them would +/// publish four records for one track and four entries in every album view, so +/// they are grouped by stem and only the highest-ranked survives. +pub fn best_derivatives(files: &[Value]) -> Vec { + let mut best: BTreeMap = BTreeMap::new(); + for f in files { + let Some(name) = f.get("name").and_then(Value::as_str) else { + continue; + }; + let Some(rank) = audio_rank(name) else { + continue; + }; + let stem = recording_stem(name); + let decoded = f + .get("track") + .and_then(|v| match v { + Value::String(s) => decode_track_number(s), + Value::Number(n) => decode_track_number(&n.to_string()), + _ => None, + }); + let track = ArchiveTrack { + file_name: name.to_string(), + title: f + .get("title") + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| name.rsplit_once('.').map(|(s, _)| s).unwrap_or(name)) + .to_string(), + track_number: decoded.map(|(_, t)| t), + disc_number: decoded.and_then(|(d, _)| d), + size: f + .get("size") + .and_then(|v| v.as_str().and_then(|s| s.parse().ok()).or_else(|| v.as_u64())), + }; + match best.get(&stem) { + Some((existing_rank, _)) if *existing_rank <= rank => {} + _ => { + best.insert(stem, (rank, track)); + } + } + } + let mut out: Vec = best.into_values().map(|(_, t)| t).collect(); + // Disc first, then track. Sorting on the track number alone would + // interleave two discs' track 1s. + out.sort_by(|a, b| { + a.disc_number + .unwrap_or(1) + .cmp(&b.disc_number.unwrap_or(1)) + .then_with(|| { + a.track_number + .unwrap_or(u32::MAX) + .cmp(&b.track_number.unwrap_or(u32::MAX)) + }) + .then_with(|| a.file_name.cmp(&b.file_name)) + }); + out +} + +pub struct InternetArchivePlugin { + http: reqwest::Client, + search_url: String, + metadata_base: String, + download_base: String, + budget: Arc, + cache: Option, + hash_cache: Option, +} + +impl Default for InternetArchivePlugin { + fn default() -> Self { + Self::new() + } +} + +impl InternetArchivePlugin { + pub fn new() -> Self { + Self::with_bases( + IA_SEARCH_URL.to_string(), + IA_METADATA_BASE.to_string(), + IA_DOWNLOAD_BASE.to_string(), + ) + } + + /// Test constructor: point every endpoint at a wiremock server. + pub fn with_bases(search_url: String, metadata_base: String, download_base: String) -> Self { + Self { + http: reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(IA_FETCH_TIMEOUT_SECS)) + .build() + .expect("build internet archive http client"), + search_url, + metadata_base: metadata_base.trim_end_matches('/').to_string(), + download_base: download_base.trim_end_matches('/').to_string(), + budget: RateBudget::new(IA_RATE_PER_SEC, IA_BURST), + cache: None, + hash_cache: None, + } + } + + fn download_url(&self, identifier: &str, file_name: &str) -> String { + format!("{}/{}/{}", self.download_base, identifier, file_name) + } + + async fn get_json(&self, url: &str, deadline: Duration) -> Option { + if self.budget.acquire(deadline).await == Lease::DeadlineExceeded { + debug!(target: "meta-music", %url, "internet archive budget deadline; degrading"); + return None; + } + let resp = match self + .http + .get(url) + .timeout(Duration::from_secs(IA_TIMEOUT_SECS)) + .send() + .await + { + Ok(r) => r, + Err(e) => { + warn!(target: "meta-music", %url, error = %e, "internet archive request failed"); + return None; + } + }; + if !resp.status().is_success() { + warn!(target: "meta-music", %url, status = %resp.status(), "internet archive non-2xx"); + return None; + } + resp.json::().await.ok() + } + + /// Build the Lucene query the advanced-search endpoint takes. + /// + /// Two things here are not obvious and were both found by probing the live + /// endpoint rather than by reading the docs: + /// + /// **1. The Live Music Archive is `mediatype:etree`, not + /// `mediatype:audio`.** A first cut asked for `mediatype:(audio)` and + /// silently excluded the entire collection — measured, that is 293,738 + /// items, including 18,334 Grateful Dead recordings, and it is the single + /// largest source of free, legally-hostable music this feeder has. The + /// search still returned *something* (a handful of `audio_music` items), so + /// nothing looked broken. + /// + /// **2. The licensing rule is pushed down into the query.** Filtering after + /// the fact wastes the whole result page on items that will be dropped: + /// `audio_music` is sorted by downloads, its most-downloaded items + /// routinely carry no `licenseurl`, and a page of those yields zero + /// records. Requiring the licence in the query means every row that comes + /// back is usable. The self-vouching collections are exempted because their + /// items frequently carry no per-item licence field at all — the collection + /// is the licence (see [`SELF_VOUCHING_COLLECTIONS`]). + fn build_query(&self, free_text: &str) -> String { + let vouching: Vec<&str> = SELF_VOUCHING_COLLECTIONS.iter().map(|(c, _)| *c).collect(); + let needs_licence: Vec<&str> = IA_COLLECTIONS + .iter() + .copied() + .filter(|c| !vouching.contains(c)) + .collect(); + + let mut clauses = vec![format!("collection:({})", vouching.join(" OR "))]; + if !needs_licence.is_empty() { + clauses.push(format!( + "(collection:({}) AND licenseurl:[* TO *])", + needs_licence.join(" OR ") + )); + } + let mut q = format!( + "mediatype:({}) AND ({})", + IA_MEDIATYPES.join(" OR "), + clauses.join(" OR ") + ); + + let text = free_text.trim(); + if !text.is_empty() { + // Quote the whole phrase: the archive's parser treats bare + // punctuation as syntax, and a music title is full of it. + let escaped = text.replace('"', " "); + q.push_str(&format!(" AND (title:(\"{escaped}\") OR creator:(\"{escaped}\"))")); + } + q + } + + /// Search items. `free_text` empty means "the browse row" — the most + /// downloaded items in the allow-listed collections. + async fn search_items( + &self, + free_text: &str, + rows: usize, + deadline: Duration, + ) -> Vec { + let url = format!( + "{}?q={}&fl%5B%5D=identifier&fl%5B%5D=title&fl%5B%5D=creator&fl%5B%5D=date\ + &fl%5B%5D=licenseurl&fl%5B%5D=collection&sort%5B%5D=downloads+desc\ + &rows={}&page=1&output=json", + self.search_url, + meta_feeder_sdk::common::percent_encode(&self.build_query(free_text)), + rows.clamp(1, 50) + ); + let Some(v) = self.get_json(&url, deadline).await else { + return Vec::new(); + }; + let docs = v + .get("response") + .and_then(|r| r.get("docs")) + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + + let mut out = Vec::new(); + for d in &docs { + let Some(identifier) = d.get("identifier").and_then(Value::as_str) else { + continue; + }; + let collections = string_list(d.get("collection")); + let licence = d + .get("licenseurl") + .and_then(Value::as_str) + .and_then(licence_from_url) + .or_else(|| collection_licence(&collections).map(str::to_string)); + // Rule 2: no per-item licence and no vouching collection ⇒ skip. + // A record with no `licence` from this tier would be a claim the + // feeder cannot back. + let Some(licence) = licence else { + debug!( + target: "meta-music", identifier, + "internet archive item has no licence and no vouching collection; skipping" + ); + continue; + }; + out.push(ArchiveItem { + identifier: identifier.to_string(), + title: first_string(d.get("title")).unwrap_or_else(|| identifier.to_string()), + creator: first_string(d.get("creator")).unwrap_or_default(), + date: first_string(d.get("date")), + licence: Some(licence), + tracks: Vec::new(), + }); + } + out + } + + /// List an item's audio files. Cached permanently — an archive item's file + /// set is effectively immutable once derived. + async fn item_tracks(&self, identifier: &str, deadline: Duration) -> Vec { + if let Some(cache) = &self.cache { + if let Ok(Some(hit)) = cache.get_ia_item(identifier) { + if let Ok(files) = serde_json::from_str::>(&hit) { + return best_derivatives(&files); + } + } + } + let url = format!("{}/{}", self.metadata_base, identifier); + let Some(v) = self.get_json(&url, deadline).await else { + return Vec::new(); + }; + let files = v + .get("files") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if let (Some(cache), Ok(json)) = (&self.cache, serde_json::to_string(&files)) { + let _ = cache.put_ia_item(identifier, &json); + } + best_derivatives(&files) + } + + /// Project one track to a wire record. + fn track_record( + &self, + item: &ArchiveItem, + track: &ArchiveTrack, + query_filters: &BTreeMap>, + ) -> DiscoveryRecord { + let mut fields: BTreeMap = BTreeMap::new(); + fields.insert("fileType".to_string(), "audio".to_string()); + fields.insert("contentKind".to_string(), "track".to_string()); + fields.insert("title".to_string(), track.title.clone()); + fields.insert("album".to_string(), item.title.clone()); + fields.insert("fileName".to_string(), track.file_name.clone()); + if let Some(ext) = track.file_name.rsplit_once('.').map(|(_, e)| e) { + fields.insert("extension".to_string(), ext.to_ascii_lowercase()); + } + if !item.creator.trim().is_empty() { + fields.insert("albumArtist".to_string(), item.creator.clone()); + fields.insert("artist".to_string(), item.creator.clone()); + fields.insert(format!("artists/{}", item.creator.trim()), "true".to_string()); + } + if let Some(n) = track.track_number { + fields.insert("trackNumber".to_string(), n.to_string()); + } + if let Some(n) = track.disc_number { + fields.insert("discNumber".to_string(), n.to_string()); + } + if let Some(sz) = track.size { + fields.insert("sizeByte".to_string(), sz.to_string()); + } + if let Some(d) = item + .date + .as_deref() + .and_then(meta_feeder_sdk::common::normalise_date) + { + fields.insert("releasedate".to_string(), d); + } + if let Some(l) = &item.licence { + fields.insert("licence".to_string(), l.clone()); + } + // The **url locator** (`0x1006`) — what makes this track playable from + // a search hit at all. + // + // A track's real content CID only exists after `compute_outcomes` + // downloads the bytes, so a search result would otherwise carry no + // `cids/*` member and the consumer would have nothing to play. + // Pre-fetching every track a row returns to fix that would mean + // downloading twenty songs to render one strip. + // + // meta-share resolves this lazily on first access: decode → fetch once + // → chunk into kubo-identical IPFS blocks → seed. After that the real + // digest outranks the locator and playback comes off bitswap. + // `METADATA_KEYS.md` §2.1 reserved this write path; this is its first + // writer. + if let Some(url_cid) = + meta_feeder_sdk::hash::compute_url_cid(&self.download_url(&item.identifier, &track.file_name)) + { + fields.insert(format!("cids/{url_cid}"), "true".to_string()); + } + + // The canonical `id` field every feeder record must carry. + fields.insert("internetarchiveid".to_string(), item.identifier.clone()); + fields.insert("source/gateway:internetarchive".to_string(), "true".to_string()); + + // The cover. + // + // ⚠ **Load-bearing for the whole tier.** The consumer's quality gate + // requires a cover, so without this every free-tier track lands in the + // raw bucket and the cold-start row — the one row whose purpose is to + // be full on a client with nothing configured — is permanently empty. + // + // An item's real artwork is an ordinary file in its file list under an + // unpredictable name; the thumbnail service is the one address that + // resolves for any item. Emitted as a `url` LOCATOR cid (rule #5 — + // never a raw `*_url` field); the gateway core upgrades it to a + // content-addressed `cover` cid when it seeds the bytes. A release + // sleeve — music vocabulary, not the video tier's `poster`. + if let Some(locator) = + meta_feeder_sdk::hash::artwork_locator(&format!("{IA_THUMB_BASE}/{}", item.identifier)) + { + fields.insert("cover".to_string(), locator); + } + + echo_filters(&mut fields, query_filters); + DiscoveryRecord { + upstream_id: "internetarchive".to_string(), + record_id: track_record_id(&item.identifier, &track.file_name), + fields, + } + } + + /// Project an item to a `pack` record — the album release as a unit. + /// + /// ⚠ Unlike meta-watch, which hides `contentKind=pack` (a season pack is a + /// nuisance; the episodes are what you watch), the music domain keeps it: + /// an album release *is* the unit people mean, and it is what a + /// download-the-whole-thing gesture acts on. + fn pack_record( + &self, + item: &ArchiveItem, + query_filters: &BTreeMap>, + ) -> DiscoveryRecord { + let mut fields: BTreeMap = BTreeMap::new(); + fields.insert("fileType".to_string(), "audio".to_string()); + fields.insert("contentKind".to_string(), "pack".to_string()); + fields.insert("title".to_string(), item.title.clone()); + fields.insert("album".to_string(), item.title.clone()); + if !item.creator.trim().is_empty() { + fields.insert("albumArtist".to_string(), item.creator.clone()); + } + fields.insert("trackCount".to_string(), item.tracks.len().to_string()); + if let Some(d) = item + .date + .as_deref() + .and_then(meta_feeder_sdk::common::normalise_date) + { + fields.insert("releasedate".to_string(), d); + } + if let Some(l) = &item.licence { + fields.insert("licence".to_string(), l.clone()); + } + fields.insert("internetarchiveid".to_string(), item.identifier.clone()); + fields.insert("source/gateway:internetarchive".to_string(), "true".to_string()); + if let Some(locator) = + meta_feeder_sdk::hash::artwork_locator(&format!("{IA_THUMB_BASE}/{}", item.identifier)) + { + fields.insert("cover".to_string(), locator); + } + // A pack is a container, not a playable file. The consumer reads this + // to grey out a play button rather than failing at byte-fetch time. + fields.insert("playable".to_string(), "false".to_string()); + echo_filters(&mut fields, query_filters); + DiscoveryRecord { + upstream_id: "internetarchive".to_string(), + record_id: item.identifier.clone(), + fields, + } + } +} + +/// Echo back structured filters the record did not already set, so the record +/// survives `record_matches` at the gateway and at meta-search. Same rule and +/// the same two exclusions as the card projection. +fn echo_filters(fields: &mut BTreeMap, query_filters: &BTreeMap>) { + for (key, allowed) in query_filters { + if key == "languages" || key == "genres" || fields.contains_key(key) || allowed.is_empty() { + continue; + } + fields.insert(key.clone(), allowed.join(",")); + } +} + +/// The archive returns some fields as a scalar and some as an array depending +/// on how many values an item has. Both shapes mean the same thing. +fn first_string(v: Option<&Value>) -> Option { + match v? { + Value::String(s) if !s.trim().is_empty() => Some(s.trim().to_string()), + Value::Array(a) => a + .iter() + .filter_map(Value::as_str) + .map(str::trim) + .find(|s| !s.is_empty()) + .map(str::to_string), + _ => None, + } +} + +fn string_list(v: Option<&Value>) -> Vec { + match v { + Some(Value::String(s)) => vec![s.to_string()], + Some(Value::Array(a)) => a + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(), + _ => Vec::new(), + } +} + +#[async_trait] +impl FeederPlugin for InternetArchivePlugin { + fn upstream_id(&self) -> &'static str { + "internetarchive" + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["audio"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &["track", "pack"] + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + self.cache = Some( + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin: "internetarchive", + source: anyhow::anyhow!("open music cache: {e}"), + })?, + ); + self.hash_cache = Some(meta_feeder_sdk::common::open_midhash_cache( + cache_dir, + "internetarchive", + )?); + Ok(()) + } + + fn health(&self) -> PluginHealth { + match self.cache { + Some(_) => PluginHealth::Ok, + None => PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + }, + } + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + + // ⚠ **An identity-anchored query this upstream cannot satisfy must + // return NOTHING**, not a free-text browse. + // + // The archive holds no MusicBrainz ids, so a query like + // `mbReleaseGroupId: fileType:audio` carries no free text for + // this plugin to search on. Falling through to the keyword path then + // means an *empty* search — which the archive answers with its + // most-downloaded items. Observed live: opening a BTS album filled its + // page with sixty Grateful Dead and shortwave-numbers-station + // recordings, every one of them a confident-looking answer to a + // question nobody asked. + // + // A consumer's album page is exactly this shape, so the failure is not + // an edge case — it is the normal detail-page query. + const IDENTITY_FILTERS: &[&str] = &[ + "mbReleaseGroupId", + "mbReleaseId", + "mbArtistId", + "mbRecordingId", + "isrc", + "tmdbid", + "imdbid", + ]; + if IDENTITY_FILTERS.iter().any(|k| query.filters.contains_key(*k)) { + debug!( + target: "meta-music", + filters = ?query.filters.keys().collect::>(), + "internet archive: id-anchored query it cannot resolve; returning nothing" + ); + return Ok(Vec::new()); + } + + let deadline = Duration::from_secs(IA_TIMEOUT_SECS); + // Item breadth is below `max_results` because one item expands into a + // whole album's worth of track records — asking for twenty items to + // fill a twenty-record page would fetch twenty file listings and + // discard most of them. + // + // ⚠ But there is a **floor**, and it is load-bearing. A first cut used + // `div_ceil(8)` alone, which asks for ONE item on a small page; an item + // whose file list turns out to hold no servable audio then yields zero + // records for the whole query, with nothing in the log to say why. The + // floor makes a single unusable item a shortfall rather than a blackout. + let item_rows = max_results.div_ceil(8).clamp(8, 24); + let items = self.search_items(query.free_text.trim(), item_rows, deadline).await; + + let mut out = Vec::with_capacity(max_results); + for mut item in items { + item.tracks = self.item_tracks(&item.identifier, deadline).await; + if item.tracks.is_empty() { + continue; + } + if wants_content_kind(query, "pack") { + out.push(self.pack_record(&item, &query.filters)); + } + if wants_content_kind(query, "track") { + for t in &item.tracks { + if out.len() >= max_results { + break; + } + out.push(self.track_record(&item, t, &query.filters)); + } + } + if out.len() >= max_results { + break; + } + } + out.truncate(max_results); + Ok(out) + } + + /// Download one track and content-address it. + /// + /// Full-store branch: real bytes, a real sha2-256 IPFS CID, seeded to + /// bitswap by the core. This is what makes the free tier genuinely part of + /// the network rather than a link to somewhere else. + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let Some((identifier, file_name)) = split_track_record_id(record_id) else { + // A bare identifier is the pack record. A pack is a container with + // no bytes of its own — its tracks are separately addressable — so + // there is nothing to resolve. + return Ok(Vec::new()); + }; + + if let Some(cache) = &self.hash_cache { + if let Some(hit) = meta_feeder_sdk::common::cached_outcome(cache, record_id, "internetarchive")? { + return Ok(hit); + } + } + + let url = self.download_url(identifier, &file_name); + if self + .budget + .acquire(Duration::from_secs(IA_TIMEOUT_SECS)) + .await + == Lease::DeadlineExceeded + { + return Err(GatewayError::RateLimited { retry_after_s: 5 }); + } + let resp = self + .http + .get(&url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("internet archive fetch: {e}")))?; + if !resp.status().is_success() { + return Err(GatewayError::Permanent(format!( + "internet archive fetch {url}: HTTP {}", + resp.status() + ))); + } + // Refuse an oversized file *before* buffering it: the core holds the + // whole outcome in memory to hash and seed it, so an unbounded body is + // an OOM on a small feeder. + if let Some(len) = resp.content_length() { + if len > IA_MAX_TRACK_BYTES { + return Err(GatewayError::Permanent(format!( + "internet archive file {file_name} is {len} bytes, over the \ + {IA_MAX_TRACK_BYTES}-byte per-track ceiling" + ))); + } + } + let bytes = resp + .bytes() + .await + .map_err(|e| GatewayError::Transient(format!("internet archive body: {e}")))?; + if bytes.len() as u64 > IA_MAX_TRACK_BYTES { + return Err(GatewayError::Permanent(format!( + "internet archive file {file_name} exceeded the per-track ceiling after download" + ))); + } + + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + if let Some(cache) = &self.hash_cache { + meta_feeder_sdk::common::store_midhash(cache, record_id, "internetarchive", &cid); + } + + // The record is rebuilt from the item so the stored metadata is + // complete even when this is reached by `compute` rather than from a + // search hit. + let deadline = Duration::from_secs(IA_TIMEOUT_SECS); + let record = { + let items = self.search_items(identifier, 1, deadline).await; + let mut item = items + .into_iter() + .find(|i| i.identifier == identifier) + .unwrap_or(ArchiveItem { + identifier: identifier.to_string(), + title: identifier.to_string(), + creator: String::new(), + date: None, + licence: None, + tracks: Vec::new(), + }); + item.tracks = self.item_tracks(identifier, deadline).await; + item.tracks + .iter() + .find(|t| t.file_name == file_name) + .map(|t| self.track_record(&item, t, &BTreeMap::new())) + }; + + Ok(vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record, + file_extension: file_name + .rsplit_once('.') + .map(|(_, e)| e.to_ascii_lowercase()), + }]) + } + + async fn handle_fetch(&self, record_id: &str) -> Result, GatewayError> { + let Some((identifier, file_name)) = split_track_record_id(record_id) else { + return Ok(None); + }; + let url = self.download_url(identifier, &file_name); + let resp = self + .http + .get(&url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("internet archive fetch: {e}")))?; + if !resp.status().is_success() { + return Ok(None); + } + let stream = futures::StreamExt::map(resp.bytes_stream(), |r| { + r.map_err(|e| GatewayError::Transient(format!("internet archive stream: {e}"))) + }); + Ok(Some(Box::pin(stream))) + } +} + +/// Does the query ask for this content kind? No filter means everything this +/// plugin serves. +fn wants_content_kind(query: &GatewayQuery, kind: &str) -> bool { + match query.filters.get("contentKind") { + None => true, + Some(values) if values.is_empty() => true, + Some(values) => values.iter().any(|v| v.trim().eq_ignore_ascii_case(kind)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn licence_urls_map_to_identifiers_not_urls() { + assert_eq!( + licence_from_url("http://creativecommons.org/licenses/by/4.0/").as_deref(), + Some("CC-BY-4.0") + ); + assert_eq!( + licence_from_url("https://creativecommons.org/licenses/by-nc-sa/3.0").as_deref(), + Some("CC-BY-NC-SA-3.0") + ); + assert_eq!( + licence_from_url("https://creativecommons.org/publicdomain/zero/1.0/").as_deref(), + Some("CC0-1.0") + ); + assert_eq!( + licence_from_url("https://creativecommons.org/publicdomain/mark/1.0/").as_deref(), + Some("PublicDomain") + ); + assert_eq!(licence_from_url(""), None); + assert_eq!(licence_from_url("https://example.com/terms"), None); + } + + /// The collections that vouch for their own items, and the one that does + /// not. `audio_music` having no fallback is what makes the `licence` field + /// a claim the feeder can actually back. + #[test] + fn only_the_allow_listed_collections_vouch_for_licensing() { + assert_eq!( + collection_licence(&["etree".to_string()]), + Some("LiveMusicArchive-TapingPolicy") + ); + assert_eq!(collection_licence(&["netlabels".to_string()]), Some("CC")); + assert_eq!(collection_licence(&["78rpm".to_string()]), Some("PublicDomain")); + assert_eq!(collection_licence(&["audio_music".to_string()]), None); + // Every vouching collection must be inside the searched allow-list — + // a row here for a collection nothing queries is dead weight, and a + // searched collection with no vouching row silently drops every item + // that lacks its own licence URL. + for (name, _) in SELF_VOUCHING_COLLECTIONS { + assert!( + IA_COLLECTIONS.contains(name), + "{name} vouches for licensing but is never searched" + ); + } + } + + /// A file name may contain `/`, and axum's `:record_id` matches exactly one + /// path segment — so a raw id would 404 on `/fetch` while working through + /// `/compute`. This round-trip is what prevents that split behaviour. + #[test] + fn record_ids_round_trip_through_a_single_path_segment() { + for name in ["Track01.flac", "disc1/Track 01.flac", "wéird ~ name.mp3"] { + let id = track_record_id("gd1977-05-08", name); + assert!(!id.contains('/'), "record id must stay one path segment: {id}"); + let (ident, back) = split_track_record_id(&id).expect("round trip"); + assert_eq!(ident, "gd1977-05-08"); + assert_eq!(back, name); + } + // A bare identifier is the pack, and has no file half. + assert!(split_track_record_id("gd1977-05-08").is_none()); + } + + /// The derivative markers the archive's own pipeline appends. Grouping on + /// the bare stem leaves each derivative in its own group, which is how the + /// first cut of this shipped four records for one track. + #[test] + fn derivative_markers_collapse_onto_one_recording_stem() { + let stem = recording_stem("Track01.flac"); + for name in [ + "Track01.mp3", + "Track01_64kb.mp3", + "Track01_128kb.mp3", + "Track01_vbr.mp3", + "Track01_sample.mp3", + ] { + assert_eq!(recording_stem(name), stem, "{name} must group with Track01"); + } + // A different recording must NOT collapse into it. + assert_ne!(recording_stem("Track02.mp3"), stem); + } + + /// One recording, four derivatives, one record. Emitting all of them would + /// show every track four times in an album view. + #[test] + fn only_the_best_derivative_of_each_recording_survives() { + let files = json!([ + {"name": "Track01.flac", "title": "Opening", "track": "1", "size": "40000000"}, + {"name": "Track01.mp3", "title": "Opening", "track": "1", "size": "8000000"}, + {"name": "Track01_64kb.mp3", "title": "Opening", "track": "1"}, + {"name": "Track02.mp3", "title": "Second", "track": "2"}, + {"name": "notes.txt"}, + {"name": "cover.jpg"} + ]); + let got = best_derivatives(files.as_array().unwrap()); + assert_eq!(got.len(), 2, "two recordings, not five files: {got:?}"); + assert_eq!(got[0].file_name, "Track01.flac", "flac outranks mp3"); + assert_eq!(got[0].track_number, Some(1)); + assert_eq!(got[1].file_name, "Track02.mp3"); + } + + /// Non-audio files are not tracks. An item's `cover.jpg` and `notes.txt` + /// must never become playable records. + #[test] + fn non_audio_files_are_ignored() { + assert!(audio_rank("cover.jpg").is_none()); + assert!(audio_rank("notes.txt").is_none()); + assert!(audio_rank("noextension").is_none()); + assert!(audio_rank("Track.FLAC").is_some(), "extension match is case-insensitive"); + // Preference order: flac beats mp3. + assert!(audio_rank("a.flac") < audio_rank("a.mp3")); + } + + /// ⚠ The Live Music Archive disc-encodes track numbers: `101` is disc 1 + /// track 1, `201` is disc 2 track 1. Read literally, a two-disc show + /// reports "track 101" through "track 212" in every UI. + #[test] + fn disc_encoded_track_numbers_are_decoded() { + assert_eq!(decode_track_number("101"), Some((Some(1), 1))); + assert_eq!(decode_track_number("115"), Some((Some(1), 15))); + assert_eq!(decode_track_number("212"), Some((Some(2), 12))); + // Plain and "n/total" ordinals stay plain. + assert_eq!(decode_track_number("7"), Some((None, 7))); + assert_eq!(decode_track_number("7/12"), Some((None, 7))); + // `100` is disc 1 track 0, which is meaningless — treat it as the + // literal ordinal rather than inventing a track zero. + assert_eq!(decode_track_number("100"), Some((None, 100))); + assert_eq!(decode_track_number("0"), None); + assert_eq!(decode_track_number(""), None); + assert_eq!(decode_track_number("abc"), None); + } + + /// Two discs' track 1s must not interleave. + #[test] + fn tracks_are_ordered_by_disc_then_track() { + let files = json!([ + {"name": "d2t01.flac", "track": "201"}, + {"name": "d1t02.flac", "track": "102"}, + {"name": "d1t01.flac", "track": "101"} + ]); + let got = best_derivatives(files.as_array().unwrap()); + assert_eq!( + got.iter().map(|t| t.file_name.as_str()).collect::>(), + vec!["d1t01.flac", "d1t02.flac", "d2t01.flac"] + ); + assert_eq!(got[2].disc_number, Some(2)); + assert_eq!(got[2].track_number, Some(1)); + } + + #[test] + fn tracks_are_ordered_by_track_number_then_name() { + let files = json!([ + {"name": "b.mp3", "track": "2"}, + {"name": "a.mp3", "track": "1"}, + {"name": "z.mp3"} + ]); + let got = best_derivatives(files.as_array().unwrap()); + assert_eq!( + got.iter().map(|t| t.file_name.as_str()).collect::>(), + vec!["a.mp3", "b.mp3", "z.mp3"], + "untracked files sort last, not first" + ); + } + + /// The archive returns single-valued fields as scalars and multi-valued + /// ones as arrays, for the same field, depending on the item. + #[test] + fn scalar_and_array_field_shapes_both_parse() { + assert_eq!( + first_string(Some(&json!("Miles Davis"))).as_deref(), + Some("Miles Davis") + ); + assert_eq!( + first_string(Some(&json!(["Miles Davis", "Gil Evans"]))).as_deref(), + Some("Miles Davis") + ); + assert_eq!(first_string(Some(&json!([" ", "Real"]))).as_deref(), Some("Real")); + assert_eq!(first_string(Some(&json!(null))), None); + assert_eq!(string_list(Some(&json!("etree"))), vec!["etree"]); + assert_eq!( + string_list(Some(&json!(["etree", "stream_only"]))), + vec!["etree", "stream_only"] + ); + } + + #[test] + fn the_query_is_scoped_to_the_collection_allow_list() { + let p = InternetArchivePlugin::new(); + let q = p.build_query(""); + for c in IA_COLLECTIONS { + assert!(q.contains(c), "collection {c} missing from the query"); + } + // Free text is added, quoted. + let q = p.build_query("kind of blue"); + assert!(q.contains("\"kind of blue\"")); + } + + /// ⚠ Regression guard for a measured, silent catalogue loss: Live Music + /// Archive items are `mediatype:etree`, and a query saying only + /// `mediatype:(audio)` matches **zero** of them (293,738 items, including + /// 18,334 Grateful Dead recordings). The search still returns a few + /// `audio_music` rows, so nothing looks broken — it just looks like a thin + /// catalogue. + #[test] + fn the_query_covers_the_etree_mediatype_not_just_audio() { + let q = InternetArchivePlugin::new().build_query(""); + assert!( + q.contains("mediatype:(audio OR etree)"), + "etree is a MEDIATYPE, not only a collection: {q}" + ); + } + + /// The licensing rule is pushed into the query, not applied after it. + /// Filtering afterwards spends the whole result page on items that will be + /// dropped — `audio_music` sorted by downloads is mostly unlicensed, so a + /// page of those yields zero records. + #[test] + fn the_licence_requirement_is_pushed_into_the_query() { + let q = InternetArchivePlugin::new().build_query(""); + assert!( + q.contains("licenseurl:[* TO *]"), + "the licence requirement must be a query clause: {q}" + ); + // …but only for the collections that do not vouch for themselves; + // etree items routinely carry no per-item licence field at all. + let vouching_clause = q + .split(" OR (collection:") + .next() + .expect("vouching clause"); + for (name, _) in SELF_VOUCHING_COLLECTIONS { + assert!( + vouching_clause.contains(name), + "{name} must be exempt from the per-item licence requirement: {q}" + ); + } + } + + #[test] + fn declares_the_audio_types_it_serves() { + let p = InternetArchivePlugin::new(); + assert_eq!(p.upstream_id(), "internetarchive"); + assert_eq!(p.served_file_types(), &["audio"]); + assert_eq!(p.served_content_kinds(), &["track", "pack"]); + } + + /// ⚠ **The bug this pins was seen live and looked like a working page.** + /// An album detail page asks `mbReleaseGroupId: fileType:audio`. The + /// archive holds no MusicBrainz ids, so that query has no free text for + /// this plugin — and an empty archive search returns its most-downloaded + /// items. A BTS album page filled with sixty Grateful Dead and + /// shortwave-numbers-station recordings, each presented as a source for + /// that album. + #[tokio::test] + async fn an_id_anchored_query_returns_nothing_rather_than_a_browse() { + let p = InternetArchivePlugin::new(); + for key in [ + "mbReleaseGroupId", + "mbReleaseId", + "mbArtistId", + "mbRecordingId", + "isrc", + ] { + let mut q = GatewayQuery::from_free_text(""); + q.filters + .insert("fileType".to_string(), vec!["audio".to_string()]); + q.filters + .insert(key.to_string(), vec!["some-id".to_string()]); + assert!( + p.handle_query(&q, 20).await.unwrap().is_empty(), + "{key} must not be answered with unrelated content" + ); + } + } + + /// …but a plain free-text audio query is still answered normally. The guard + /// must not disable the plugin. + #[tokio::test] + async fn a_plain_free_text_query_is_not_blocked_by_the_guard() { + let p = InternetArchivePlugin::new(); + let mut q = GatewayQuery::from_free_text("grateful dead"); + q.filters + .insert("fileType".to_string(), vec!["audio".to_string()]); + // No network here, so this resolves to an empty result — the assertion + // is that it took the search path rather than the early return, which + // the id-filter test above covers by contrast. + let _ = p.handle_query(&q, 5).await.unwrap(); + } + + /// A pack has no bytes of its own — its tracks are separately addressable. + #[tokio::test] + async fn a_pack_record_id_resolves_to_no_outcomes() { + let p = InternetArchivePlugin::new(); + assert!(p.compute_outcomes("some-identifier").await.unwrap().is_empty()); + } + + fn item() -> ArchiveItem { + ArchiveItem { + identifier: "gd1977-05-08".into(), + title: "Barton Hall".into(), + creator: "Grateful Dead".into(), + date: Some("1977-05-08".into()), + licence: Some("LiveMusicArchive-TapingPolicy".into()), + tracks: vec![ArchiveTrack { + file_name: "gd77-05-08d1t01.flac".into(), + title: "New Minglewood Blues".into(), + track_number: Some(1), + disc_number: Some(1), + size: Some(50_000_000), + }], + } + } + + #[test] + fn a_track_record_carries_the_music_vocabulary_and_a_licence() { + let p = InternetArchivePlugin::new(); + let it = item(); + let r = p.track_record(&it, &it.tracks[0], &BTreeMap::new()); + assert_eq!(r.fields.get("fileType").map(String::as_str), Some("audio")); + assert_eq!(r.fields.get("contentKind").map(String::as_str), Some("track")); + assert_eq!(r.fields.get("album").map(String::as_str), Some("Barton Hall")); + assert_eq!( + r.fields.get("albumArtist").map(String::as_str), + Some("Grateful Dead") + ); + assert_eq!(r.fields.get("trackNumber").map(String::as_str), Some("1")); + assert_eq!(r.fields.get("extension").map(String::as_str), Some("flac")); + assert_eq!( + r.fields.get("licence").map(String::as_str), + Some("LiveMusicArchive-TapingPolicy") + ); + assert_eq!( + r.fields.get("internetarchiveid").map(String::as_str), + Some("gd1977-05-08") + ); + // ⚠ The url locator, without which this track cannot be played from a + // search hit — a content digest does not exist until the bytes are + // fetched, which happens long after the row is rendered. + let want = meta_feeder_sdk::hash::compute_url_cid( + "https://archive.org/download/gd1977-05-08/gd77-05-08d1t01.flac", + ) + .expect("locator"); + assert_eq!(r.fields.get(&format!("cids/{want}")).map(String::as_str), Some("true")); + // ⚠ A cover IS claimed, and must be: without one the gate hides every + // free-tier track and the cold-start row is permanently empty. It is a + // `url` LOCATOR cid, not a raw URL — rule #5. + let want_cover = meta_feeder_sdk::hash::compute_url_cid( + "https://archive.org/services/img/gd1977-05-08", + ) + .expect("cover locator"); + assert_eq!(r.fields.get("cover").map(String::as_str), Some(want_cover.as_str())); + assert!( + !r.fields.contains_key("cover_url"), + "artwork must never be a raw *_url field" + ); + // Never a video field. + assert!(r.fields.get("videoType").is_none()); + assert!(r.fields.get("tmdbid").is_none()); + } + + /// The album release stays first class here, unlike a season pack in the + /// video domain — but it is explicitly non-playable, so the consumer greys + /// the play button rather than failing at byte-fetch time. + #[test] + fn a_pack_record_is_emitted_and_marked_unplayable() { + let p = InternetArchivePlugin::new(); + let r = p.pack_record(&item(), &BTreeMap::new()); + assert_eq!(r.fields.get("contentKind").map(String::as_str), Some("pack")); + assert_eq!(r.fields.get("playable").map(String::as_str), Some("false")); + assert_eq!(r.fields.get("trackCount").map(String::as_str), Some("1")); + assert_eq!(r.record_id, "gd1977-05-08"); + } + + #[test] + fn filters_are_echoed_except_the_key_sets() { + let p = InternetArchivePlugin::new(); + let it = item(); + let mut filters = BTreeMap::new(); + filters.insert("licence".to_string(), vec!["cc".to_string()]); + filters.insert("genres".to_string(), vec!["rock".to_string()]); + let r = p.track_record(&it, &it.tracks[0], &filters); + // The record's own licence wins over the echo. + assert_eq!( + r.fields.get("licence").map(String::as_str), + Some("LiveMusicArchive-TapingPolicy") + ); + assert!(r.fields.get("genres").is_none()); + } +} diff --git a/feeder-plugin/internetarchive-feeder/src/lib.rs b/feeder-plugin/internetarchive-feeder/src/lib.rs new file mode 100644 index 0000000..5509fc1 --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/src/lib.rs @@ -0,0 +1,6 @@ +//! `internetarchive-feeder` library surface. +//! +//! Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless. + +pub mod consts; +pub mod internetarchive; diff --git a/feeder-plugin/internetarchive-feeder/src/main.rs b/feeder-plugin/internetarchive-feeder/src/main.rs new file mode 100644 index 0000000..98401d2 --- /dev/null +++ b/feeder-plugin/internetarchive-feeder/src/main.rs @@ -0,0 +1,33 @@ +//! `internetarchive-feeder` — a single-upstream feeder sidecar. +//! +//! Internet Archive free audio (etree, netlabels, 78rpm). Open, keyless. +//! +//! Upstream: Internet Archive (archive.org) +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use meta_feeder_sdk::plugin::FeederPlugin; +use meta_feeder_sdk::serve_feeders; +use internetarchive_feeder::internetarchive::InternetArchivePlugin; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(InternetArchivePlugin::new())]; + serve_feeders(plugins, state_dir, listen).await +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]