From 5e4581109d0c76c4845254f359bf88b484d966d6 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 24 Aug 2026 19:08:08 +0200 Subject: [PATCH] feeder rework --- .gitattributes | 6 + .github/workflows/publish.yml | 59 + Cargo.lock | 1948 +++++++++++++++++++ Cargo.toml | 9 + README.md | 18 +- feeder-plugin/jamendo-feeder/Cargo.toml | 42 + feeder-plugin/jamendo-feeder/Dockerfile | 44 + feeder-plugin/jamendo-feeder/src/consts.rs | 104 + feeder-plugin/jamendo-feeder/src/jamendo.rs | 1473 ++++++++++++++ feeder-plugin/jamendo-feeder/src/lib.rs | 7 + feeder-plugin/jamendo-feeder/src/main.rs | 33 + feeder-plugin/jamendo-feeder/src/shared.rs | 150 ++ rust-toolchain.toml | 3 + 13 files changed, 3895 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/publish.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 feeder-plugin/jamendo-feeder/Cargo.toml create mode 100644 feeder-plugin/jamendo-feeder/Dockerfile create mode 100644 feeder-plugin/jamendo-feeder/src/consts.rs create mode 100644 feeder-plugin/jamendo-feeder/src/jamendo.rs create mode 100644 feeder-plugin/jamendo-feeder/src/lib.rs create mode 100644 feeder-plugin/jamendo-feeder/src/main.rs create mode 100644 feeder-plugin/jamendo-feeder/src/shared.rs create mode 100644 rust-toolchain.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..1d6dfe7 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/paper-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/jamendo-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..96f465c --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1948 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jamendo-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "meta-feeder-sdk", + "redb", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..90a5fd9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-jamendo" diff --git a/README.md b/README.md index 72998de..49339da 100644 --- a/README.md +++ b/README.md @@ -1 +1,17 @@ -# meta-feeder-jamendo \ No newline at end of file +# meta-feeder-jamendo + +Jamendo Creative-Commons music. Opt-in: needs a client id. + +**Upstream:** Jamendo API v3.0 (api.jamendo.com) +**Role:** bytes + +A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin` +from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk). + +```bash +cargo build --release --bin jamendo-feeder +cargo test +``` + +Scope and naming conventions: `docs/project-architecture/feeder-architecture.md` +in the meta-root. diff --git a/feeder-plugin/jamendo-feeder/Cargo.toml b/feeder-plugin/jamendo-feeder/Cargo.toml new file mode 100644 index 0000000..486d153 --- /dev/null +++ b/feeder-plugin/jamendo-feeder/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "jamendo-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Jamendo Creative-Commons music. Opt-in: needs a client id. +# Upstream: Jamendo API v3.0 (api.jamendo.com) + +[lib] +name = "jamendo_feeder" +path = "src/lib.rs" + +[[bin]] +name = "jamendo-feeder" +path = "src/main.rs" + +[dependencies] +# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to: +# meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "" } +meta-feeder-sdk = { path = "../../../meta-feeder-sdk" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +bytes = "1" +futures = "0.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +redb = "2" +base64 = "0.22" + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/jamendo-feeder/Dockerfile b/feeder-plugin/jamendo-feeder/Dockerfile new file mode 100644 index 0000000..1f6f4db --- /dev/null +++ b/feeder-plugin/jamendo-feeder/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# jamendo-feeder sidecar. Multi-stage build; build context is this repo's root. +# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied +# in — `git` must be present in the builder for cargo to fetch it. + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=jamendo-feeder-target,target=/build/target \ + cargo build --release --bin jamendo-feeder \ + && cp target/release/jamendo-feeder /jamendo-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /jamendo-feeder /usr/local/bin/jamendo-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/jamendo-feeder"] diff --git a/feeder-plugin/jamendo-feeder/src/consts.rs b/feeder-plugin/jamendo-feeder/src/consts.rs new file mode 100644 index 0000000..7190c74 --- /dev/null +++ b/feeder-plugin/jamendo-feeder/src/consts.rs @@ -0,0 +1,104 @@ +//! Constants for `jamendo-feeder`. + +/// User-Agent for outbound HTTP. +/// +/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent +/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to +/// misdiagnose. Identify the application and carry contact information. +pub const USER_AGENT: &str = concat!( + "jamendo-feeder/", + env!("CARGO_PKG_VERSION"), + " ( https://github.com/worph/meta-feeder-jamendo )" +); + +/// Jamendo web API v3 base. +pub const JAMENDO_API_BASE: &str = "https://api.jamendo.com/v3.0"; +/// Streaming/download format segment asked of the API. +/// +/// `mp32` is Jamendo's high-bitrate MP3 derivative. Deliberately not `flac`: +/// this tier's bytes are fetched, buffered whole in the gateway core, and then +/// seeded to every peer that asks — an order-of-magnitude size increase for a +/// quality difference no one has asked for. +pub const JAMENDO_AUDIO_FORMAT: &str = "mp32"; +/// Sustained Jamendo request rate. +/// +/// Jamendo publishes no per-second ceiling — only the monthly quota below — so +/// this is a courtesy limit, not a documented one. It is deliberately modest: +/// the binding constraint here is the *month*, and a burst that fills a page +/// faster buys nothing but a nearer exhaustion date. +pub const JAMENDO_RATE_PER_SEC: f64 = 2.0; +/// Jamendo burst ceiling. +pub const JAMENDO_BURST: f64 = 4.0; +/// ⚠ **The published non-commercial quota: 35,000 API requests per MONTH.** +/// +/// This is the one number that makes Jamendo unlike every other upstream in +/// this crate, and getting it wrong is not recoverable by waiting a second. +/// MusicBrainz's 1/s is a *rate*: exceed it and you are throttled, back off and +/// you continue. This is a *budget*: spend it and the upstream is gone until +/// the calendar turns. +/// +/// A token bucket cannot express that — it refills. So this tier carries a +/// **persisted monthly counter** in the plugin cache (`MusicCache:: +/// jamendo_quota_spend`) alongside the per-second bucket, and refuses calls +/// rather than discovering the ceiling from a 429 three weeks in. +pub const JAMENDO_MONTHLY_QUOTA: u64 = 35_000; +/// Default share of [`JAMENDO_MONTHLY_QUOTA`] this feeder will spend. +/// +/// Headroom is deliberate and serves two distinct purposes: +/// +/// 1. The counter is **per feeder instance**, but the quota is per *client_id*. +/// Two gateways configured with the same credential each believe they own +/// the whole month. Until the counter is shared (it would have to live in +/// meta-core to be), the margin is what keeps a two-peer deployment from +/// silently overspending. +/// 2. It leaves a reserve for `compute_outcomes` — the calls a user is +/// actually waiting on — after a month of browse rows. +pub const JAMENDO_DEFAULT_MONTHLY_BUDGET: u64 = 30_000; +/// Request timeout for a Jamendo API call (seconds). +pub const JAMENDO_TIMEOUT_SECS: u64 = 20; +/// Budget-permit deadline for a query somebody is waiting on. +pub const JAMENDO_WAIT_DEADLINE_SECS: u64 = 20; +/// Timeout for downloading one track's bytes (seconds). +pub const JAMENDO_FETCH_TIMEOUT_SECS: u64 = 300; +/// Hard ceiling on the bytes this tier will pull for a single track. +/// +/// Lower than the Internet Archive's 200 MB because this tier fetches MP3 +/// only, never a lossless master. A 100 MB MP3 is roughly a three-hour +/// recording — past any plausible track and into "someone published a DJ set +/// as one file" territory, which the gateway core would buffer whole. +pub const JAMENDO_MAX_TRACK_BYTES: u64 = 100 * 1024 * 1024; +/// Max `limit` the API accepts on one page. +pub const JAMENDO_PAGE_MAX: usize = 200; +/// Ordering used for the keyword-less browse row (the free-tier home row). +/// +/// The consumer's cold-start row carries no text, exactly like the Internet +/// Archive tier's. Jamendo answers an order-only query with its catalogue in +/// popularity order, which is what that row wants. +pub const JAMENDO_BROWSE_ORDER: &str = "popularity_total_desc"; +/// Extra attempts made when Jamendo answers a query with an empty page. +/// +/// ⚠ **Not defensive padding — a measured mitigation for an upstream that +/// intermittently lies.** Jamendo answers a perfectly good query with +/// `HTTP 200`, `headers.status: "success"` and `results_count: 0` a large +/// fraction of the time. Measured 2026-08-21 by firing one unchanged URL +/// repeatedly: +/// +/// | query shape | empty responses | +/// |---|---| +/// | `search=love` (×12) | 2 | +/// | `namesearch=love` (×8) | 3 | +/// | `order=popularity_total_desc` (×8) | **4** | +/// +/// It looks like one replica behind their load balancer serving an empty +/// index. Nothing in the envelope distinguishes it from a genuine no-match, so +/// a single-attempt client turns a third of every Jamendo-backed row into a +/// blank — and the consumer's sticky per-row merge would quietly paper over it +/// as "the row is sometimes short". +/// +/// Two retries take the odds of a false empty from ~1-in-3 to ~1-in-27. The +/// quota cost is bounded because a retry only happens on an empty page: +/// roughly 0.5 extra calls per query at the measured rate. +pub const JAMENDO_EMPTY_RETRIES: u32 = 2; +/// Pause between the attempts above. Long enough to land on a different +/// backend, short enough that three attempts stay inside a search timeout. +pub const JAMENDO_EMPTY_RETRY_DELAY_MS: u64 = 250; diff --git a/feeder-plugin/jamendo-feeder/src/jamendo.rs b/feeder-plugin/jamendo-feeder/src/jamendo.rs new file mode 100644 index 0000000..0cf3a7a --- /dev/null +++ b/feeder-plugin/jamendo-feeder/src/jamendo.rs @@ -0,0 +1,1473 @@ +//! The `jamendo` upstream — the **contemporary** half of the free content tier. +//! +//! `internetarchive` already proves a client can play music with no credential +//! configured. What it cannot do is sound like this century: its allow-listed +//! collections are live taper recordings (`etree`), net-label back-catalogue and +//! digitised 78s. Jamendo is ~600k Creative-Commons tracks by working artists, +//! with cover art, genres and release dates — the same shape of catalogue a +//! commercial service has, licensed for redistribution. +//! +//! ## Why this tier exists despite Jamendo's API terms +//! +//! **Read this before "simplifying" the gate below.** Jamendo's API Terms of +//! Use, clause 3, say verbatim: +//! +//! > "Applications must not be specifically designed to cache the content nor +//! > offering an offline access to the content. Caching system may only be used +//! > to the extent reasonably necessary for the operation of the Application. +//! > Developers commit to ensure that their applications reflect changes made +//! > to the content as soon as reasonably possible." +//! +//! The gateway's auto-store does the opposite of all three: it PUTs the bytes +//! into meta-core and seeds them to any peer that asks, permanently, at a cid +//! that by construction *cannot* reflect an upstream change. On the face of it +//! that is the same conflict `meta-listen/ARCHITECTURE.md` §4.4 uses to reject +//! SoundCloud. +//! +//! It is not the same, and the difference is the whole justification for this +//! file: +//! +//! - **SoundCloud** hosts all-rights-reserved audio. There is no licence to +//! redistribute under, so rehosting is infringement and the terms are the +//! least of it. +//! - **Jamendo** publishes every track under a Creative Commons licence +//! (clause 3.1), and *all six* CC variants permit redistribution of verbatim +//! copies. That grant runs from the artist to the world; Jamendo is not a +//! party to it and cannot withdraw it. +//! +//! So the exposure here is **contractual, not copyright**: the realistic +//! downside is losing the `client_id`, not a takedown. That is a risk an +//! operator can knowingly accept, which is why the credential is per-deployment +//! configuration rather than something baked in. +//! +//! Two obligations follow into the code and must not be dropped: +//! +//! 1. **Only tracks whose rights are legible.** A record is emitted only when +//! `license_ccurl` resolves to a real CC identifier **and** the artist has +//! switched free download on (`audiodownload_allowed`). See +//! [`redistributable`]. No licence ⇒ no record, exactly as the Internet +//! Archive tier treats an item with no `licenseurl`. +//! 2. **Attribution (clause 4.1).** Every record carries the artist credit and +//! a `sourceUrl` backlink to the track's page on jamendo.com — the one +//! URL-shaped field `METADATA_KEYS.md` §5 sanctions, because it is +//! click-through metadata rather than a byte reference. +//! +//! ## The month is the budget, not the second +//! +//! Every other upstream in this crate is limited by a *rate*. Jamendo's +//! non-commercial plan is limited by a **monthly quota of 35,000 requests**, +//! and that difference is load-bearing: a token bucket refills, a month does +//! not. A browse row on a refresh timer will quietly eat the quota and the tier +//! then goes dark until the calendar turns, with nothing in the log to connect +//! the two events weeks apart. Hence the persisted ledger in +//! [`MidhashCache::jamendo_quota_spend`] and the deliberate headroom in +//! `consts::JAMENDO_DEFAULT_MONTHLY_BUDGET`. +//! +//! ## Three traps found by probing the live API, not by reading the docs +//! +//! 1. **`audio` is a signed, expiring URL** — it carries a `from=` +//! query parameter. Wrapping it in a `url` locator cid would mint a cid that +//! is unresolvable within the hour *and* a different cid on every request +//! for the same track, poisoning every cache downstream. The locator is +//! always built from `audiodownload`, which is a clean, stable, keyless +//! path. [`track_download_url`] is the only place that decision lives. +//! 2. **The download URL lies about its content type.** It answers +//! `content-type: text/html` for what is unambiguously an MP3 (verified: +//! the body begins `ID3`, and `content-disposition` names a `.mp3` file). +//! Nothing here may branch on the response content type. +//! 3. **The upstream intermittently answers a good query with an empty page** — +//! `HTTP 200`, `headers.status: "success"`, `results_count: 0`, for a URL +//! that returned twenty results a second earlier. Measured at 2-in-12 for a +//! text search and **4-in-8** for a browse row. Nothing distinguishes it +//! from a genuine no-match, so `get_results` retries an empty success (and +//! only an empty success — never an error envelope). Without that, a third +//! of every Jamendo-backed row would blank at random and the consumer's +//! sticky per-row merge would hide it as "sometimes short". +//! 4. **Cover URLs cannot be normalised.** `album_image` looks per-album but +//! carries a `trackid` parameter, so two tracks on one album yield two cid +//! for identical bytes — tempting to strip. Measured: for album 570388, +//! dropping `trackid` returns a placeholder `image/gif` rather than the +//! sleeve, while for album 136930 it returns the real JPEG. The parameter is +//! load-bearing for some albums and not others, with no way to tell in +//! advance, so the API's own URL is used verbatim and the duplicate seeding +//! is accepted. + +use std::collections::BTreeMap; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use meta_feeder_sdk::cache::MidhashCache; +use meta_feeder_sdk::config::{ConfigField as F, ConfigSchema}; +use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, GatewayQuery, HashKind, HashOutcome}; +use meta_feeder_sdk::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; +use serde_json::Value; +use tracing::{debug, warn}; + +use crate::consts::{ + JAMENDO_API_BASE, JAMENDO_AUDIO_FORMAT, JAMENDO_BROWSE_ORDER, JAMENDO_BURST, + JAMENDO_DEFAULT_MONTHLY_BUDGET, JAMENDO_EMPTY_RETRIES, JAMENDO_EMPTY_RETRY_DELAY_MS, + JAMENDO_FETCH_TIMEOUT_SECS, JAMENDO_MAX_TRACK_BYTES, + JAMENDO_MONTHLY_QUOTA, JAMENDO_PAGE_MAX, JAMENDO_RATE_PER_SEC, JAMENDO_TIMEOUT_SECS, + JAMENDO_WAIT_DEADLINE_SECS, USER_AGENT, +}; +use meta_feeder_sdk::licence_from_url; +use meta_feeder_sdk::budget::{Lease, RateBudget}; +use crate::shared::{echo_filters, is_identity_anchored, wants_content_kind}; + +// --------------------------------------------------------------------------- +// The month key +// --------------------------------------------------------------------------- + +/// The UTC `YYYY-MM` the quota ledger is keyed by. +/// +/// Hand-rolled rather than pulling in `chrono`/`time` for one formatted string: +/// this crate has no date dependency and the civil-from-days conversion is a +/// dozen well-known lines. Days-since-epoch → civil date is Howard Hinnant's +/// algorithm, shifted so the era starts in March and the leap day lands at the +/// end of a year. +/// +/// ⚠ **UTC, deliberately.** A local-time key would roll the month at a +/// different instant than Jamendo's own accounting and would move under a +/// container that changes `TZ`. +pub fn utc_year_month(now: SystemTime) -> String { + let secs = now.duration_since(UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0); + let days = (secs / 86_400) as i64; + let (y, m) = civil_year_month(days); + format!("{y:04}-{m:02}") +} + +/// Days since 1970-01-01 → `(year, month)`. +fn civil_year_month(days_since_epoch: i64) -> (i64, u32) { + // Shift the epoch to 0000-03-01 so February (and its leap day) is the last + // month of the era, which removes every leap-year special case below. + let z = days_since_epoch + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = (z - era * 146_097) as u64; // [0, 146096] + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; // [0, 399] + let y = yoe as i64 + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); // [0, 365] + let mp = (5 * doy + 2) / 153; // [0, 11], March = 0 + let m = if mp < 10 { mp + 3 } else { mp - 9 }; // [1, 12] + (if m <= 2 { y + 1 } else { y }, m as u32) +} + +// --------------------------------------------------------------------------- +// Config +// --------------------------------------------------------------------------- + +/// Operator config. +/// +/// Unlike every other upstream in this crate, this one **requires** a +/// credential: Jamendo answers nothing without a `client_id`. A feeder with no +/// key soft-skips the plugin at startup (`ConfigError::MissingConfig`) rather +/// than failing the whole binary — the other three upstreams keep answering, so +/// a cold client still shows a populated board. +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct JamendoConfig { + #[serde(default)] + pub client_id: Option, + #[serde(default)] + pub monthly_request_budget: Option, +} + +impl JamendoConfig { + /// Env seed, used on first boot before any `config.json` exists. + pub fn from_env() -> Self { + Self { + client_id: std::env::var("JAMENDO_CLIENT_ID") + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()), + monthly_request_budget: std::env::var("JAMENDO_MONTHLY_REQUEST_BUDGET") + .ok() + .and_then(|v| v.parse().ok()), + } + } +} + +// --------------------------------------------------------------------------- +// The upstream record +// --------------------------------------------------------------------------- + +/// One Jamendo track, reduced to what a record needs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JamendoTrack { + pub id: String, + pub name: String, + pub artist_name: String, + pub album_name: String, + pub album_id: String, + pub position: Option, + pub duration_secs: Option, + pub releasedate: Option, + /// Resolved SPDX-ish identifier, never the raw URL. + pub licence: String, + pub download_url: String, + pub cover_url: Option, + pub share_url: Option, + pub genres: Vec, + pub lang: Option, +} + +/// The stable, keyless byte URL for a track. +/// +/// ⚠ **Never `audio`.** That field is a signed URL with an expiring `from=` +/// token: a locator cid minted from it stops resolving within the hour, and +/// because a cid is a hash of the URL string, a fresh token means a fresh cid +/// for the same track on every single query. `audiodownload` is a plain path +/// (`/download/track///`) that needs no credential and does not +/// rotate. +fn track_download_url(raw: &Value) -> Option { + let url = raw.get("audiodownload").and_then(Value::as_str)?.trim(); + if url.is_empty() || url.contains("from=") { + return None; + } + Some(url.to_string()) +} + +/// May this feeder host bytes for this track? Returns the licence identifier +/// to stamp on the record. +/// +/// Both halves are required and neither is a formality: +/// +/// - **`license_ccurl` must resolve.** A record from a free tier with no +/// `licence` is a claim the feeder cannot back (`METADATA_KEYS.md` +/// §`licence`: absence means "unknown", never "unrestricted"). +/// - **`audiodownload_allowed` must be true.** It is the artist's own switch +/// for "you may take a copy of this file". A CC licence permits +/// redistribution in the abstract; this is the upload deciding whether the +/// file is offered at all, and taking bytes the artist has not offered is +/// exactly what an operator accepting the API-terms risk did *not* sign up +/// for. +pub fn redistributable(raw: &Value) -> Option { + if !raw + .get("audiodownload_allowed") + .and_then(Value::as_bool) + .unwrap_or(false) + { + return None; + } + let url = raw.get("license_ccurl").and_then(Value::as_str)?; + licence_from_url(url) +} + +/// Parse one API result object. `None` for anything this tier may not serve. +pub fn parse_track(raw: &Value) -> Option { + let id = raw.get("id").and_then(Value::as_str)?.trim(); + let name = raw.get("name").and_then(Value::as_str)?.trim(); + if id.is_empty() || name.is_empty() { + return None; + } + let licence = redistributable(raw)?; + let download_url = track_download_url(raw)?; + + let genres = raw + .get("musicinfo") + .and_then(|m| m.get("tags")) + .and_then(|t| t.get("genres")) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + + Some(JamendoTrack { + id: id.to_string(), + name: name.to_string(), + artist_name: str_field(raw, "artist_name").unwrap_or_default(), + album_name: str_field(raw, "album_name").unwrap_or_default(), + album_id: str_field(raw, "album_id").unwrap_or_default(), + // `position` is a JSON number here and a string elsewhere in Jamendo's + // own responses; accept both rather than silently losing track order. + position: raw.get("position").and_then(|v| match v { + Value::Number(n) => n.as_u64().map(|n| n as u32), + Value::String(s) => s.trim().parse().ok(), + _ => None, + }), + duration_secs: raw.get("duration").and_then(|v| match v { + Value::Number(n) => n.as_u64(), + Value::String(s) => s.trim().parse().ok(), + _ => None, + }), + releasedate: str_field(raw, "releasedate"), + licence, + download_url, + cover_url: str_field(raw, "album_image").or_else(|| str_field(raw, "image")), + share_url: str_field(raw, "shareurl"), + genres, + lang: raw + .get("musicinfo") + .and_then(|m| m.get("lang")) + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string), + }) +} + +fn str_field(v: &Value, key: &str) -> Option { + v.get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) +} + +// --------------------------------------------------------------------------- +// The plugin +// --------------------------------------------------------------------------- + +pub struct JamendoPlugin { + http: reqwest::Client, + api_base: String, + config: JamendoConfig, + client_id: Option, + monthly_budget: u64, + budget: Arc, + cache: Option, + hash_cache: Option, +} + +impl Default for JamendoPlugin { + fn default() -> Self { + Self::new() + } +} + +impl JamendoPlugin { + pub fn new() -> Self { + Self::with_base(JAMENDO_API_BASE.to_string()) + } + + /// Test constructor: point the API at a wiremock server. + pub fn with_base(api_base: String) -> Self { + Self { + http: reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(JAMENDO_FETCH_TIMEOUT_SECS)) + .build() + .expect("build jamendo http client"), + api_base: api_base.trim_end_matches('/').to_string(), + config: JamendoConfig::from_env(), + client_id: None, + monthly_budget: JAMENDO_DEFAULT_MONTHLY_BUDGET, + budget: RateBudget::new(JAMENDO_RATE_PER_SEC, JAMENDO_BURST), + cache: None, + hash_cache: None, + } + } + + /// File wins over env, exactly like every other feeder. No hot reload — + /// bounce the feeder. + fn load_config(&mut self, cache_dir: &Path) { + let path = cache_dir.join("config.json"); + if let Ok(bytes) = std::fs::read(&path) { + if let Ok(file_cfg) = serde_json::from_slice::(&bytes) { + if let Some(id) = file_cfg.client_id.map(|s| s.trim().to_string()) { + if !id.is_empty() { + self.config.client_id = Some(id); + } + } + if file_cfg.monthly_request_budget.is_some() { + self.config.monthly_request_budget = file_cfg.monthly_request_budget; + } + } + } + } + + /// Take one call off this month's budget. + /// + /// Returns `false` when the month is spent, in which case the caller must + /// degrade (empty result, loud log) rather than call the API anyway. With + /// no cache open the ledger cannot be consulted, and an unmetered upstream + /// with a hard monthly ceiling is worse than an absent one — so that case + /// refuses too. + fn spend_quota(&self) -> bool { + let Some(cache) = &self.cache else { + warn!(target: "meta-music", "jamendo: no cache open, cannot meter the monthly quota; refusing"); + return false; + }; + let month = utc_year_month(SystemTime::now()); + match cache.jamendo_quota_spend(&month, self.monthly_budget) { + Ok(Some(spent)) => { + // One line an operator can grep for when the tier goes quiet at + // the end of a month. + if spent % 500 == 0 || spent + 50 > self.monthly_budget { + debug!( + target: "meta-music", month = %month, spent, + budget = self.monthly_budget, "jamendo monthly quota" + ); + } + true + } + Ok(None) => { + warn!( + target: "meta-music", month = %month, budget = self.monthly_budget, + "jamendo monthly request budget exhausted; tier is dark until the month rolls" + ); + false + } + Err(e) => { + warn!(target: "meta-music", error = %e, "jamendo quota ledger unreadable; refusing"); + false + } + } + } + + /// GET a metered Jamendo API call. `None` on any failure — this tier + /// degrades, it never fails a whole gateway query. + async fn get_json(&self, url: &str) -> Option { + if self + .budget + .acquire(Duration::from_secs(JAMENDO_WAIT_DEADLINE_SECS)) + .await + == Lease::DeadlineExceeded + { + debug!(target: "meta-music", "jamendo rate budget deadline; degrading"); + return None; + } + if !self.spend_quota() { + return None; + } + let resp = match self + .http + .get(url) + .timeout(Duration::from_secs(JAMENDO_TIMEOUT_SECS)) + .send() + .await + { + Ok(r) => r, + Err(e) => { + warn!(target: "meta-music", error = %e, "jamendo request failed"); + return None; + } + }; + if !resp.status().is_success() { + warn!(target: "meta-music", status = %resp.status(), "jamendo non-2xx"); + return None; + } + let body = resp.json::().await.ok()?; + + // ⚠ Jamendo answers an invalid credential, an exhausted quota and a + // malformed parameter with **HTTP 200** and an error in the envelope. + // Trusting the status code alone turns a dead client_id into a silently + // empty tier — the exact "looks fine, returns nothing" failure this + // crate keeps paying for elsewhere. + let status_ok = body + .get("headers") + .and_then(|h| h.get("status")) + .and_then(Value::as_str) + .is_some_and(|s| s.eq_ignore_ascii_case("success")); + if !status_ok { + let msg = body + .get("headers") + .and_then(|h| h.get("error_message")) + .and_then(Value::as_str) + .unwrap_or("(no message)"); + let code = body + .get("headers") + .and_then(|h| h.get("code")) + .map(|c| c.to_string()) + .unwrap_or_default(); + warn!(target: "meta-music", code = %code, message = %msg, "jamendo API error envelope"); + return None; + } + Some(body) + } + + fn results(body: &Value) -> Vec { + body.get("results") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default() + } + + /// GET a page of results, retrying an **empty but successful** answer. + /// + /// ⚠ Jamendo intermittently answers a good query with `HTTP 200`, + /// `status: "success"` and zero results — measured at up to 4-in-8 for a + /// browse row (`consts::JAMENDO_EMPTY_RETRIES`). Nothing in the envelope + /// distinguishes that from a genuine no-match, so the only available + /// signal is that asking again usually works. + /// + /// Only an *empty success* is retried. An error envelope (a dead + /// `client_id`, a malformed parameter) is returned immediately: retrying + /// those would triple the quota burn on a request that cannot succeed. + async fn get_results(&self, url: &str) -> Vec { + for attempt in 0..=JAMENDO_EMPTY_RETRIES { + let Some(body) = self.get_json(url).await else { + // Hard failure or error envelope — already logged, do not retry. + return Vec::new(); + }; + let results = Self::results(&body); + if !results.is_empty() { + if attempt > 0 { + debug!( + target: "meta-music", attempt, + "jamendo: a retry rescued a page the upstream had answered empty" + ); + } + return results; + } + if attempt < JAMENDO_EMPTY_RETRIES { + tokio::time::sleep(Duration::from_millis(JAMENDO_EMPTY_RETRY_DELAY_MS)).await; + } + } + debug!( + target: "meta-music", attempts = JAMENDO_EMPTY_RETRIES + 1, + "jamendo: empty page on every attempt; treating it as a genuine no-match" + ); + Vec::new() + } + + /// Build a `/tracks/` URL for a query. + fn tracks_url(&self, query: &GatewayQuery, limit: usize) -> Option { + let client_id = self.client_id.as_ref()?; + let enc = meta_feeder_sdk::common::percent_encode; + let mut url = format!( + "{}/tracks/?client_id={}&format=json&limit={}&include={}&audioformat={}", + self.api_base, + enc(client_id), + limit.clamp(1, JAMENDO_PAGE_MAX), + enc("licenses musicinfo"), + JAMENDO_AUDIO_FORMAT, + ); + + let text = query.free_text.trim(); + if text.is_empty() { + // The keyword-less browse row: the catalogue in popularity order. + url.push_str(&format!("&order={JAMENDO_BROWSE_ORDER}")); + } else { + // `search` spans track, album and artist name plus tags — the + // closest match to a single consumer search box. `namesearch` would + // only look at the track title and miss every artist query. + url.push_str(&format!("&search={}", enc(text))); + } + + // Push a `genres:` filter down into the API rather than fetching a + // popularity page and discarding most of it at `record_matches`. The + // gateway would drop those records anyway; asking the upstream for the + // right ones is the difference between a full row and an empty one. + if let Some(genres) = query.filters.get("genres") { + let tags: Vec = genres + .iter() + .map(|g| g.trim().to_ascii_lowercase()) + .filter(|g| !g.is_empty()) + .collect(); + if !tags.is_empty() { + url.push_str(&format!("&tags={}", enc(&tags.join(" ")))); + } + } + Some(url) + } + + /// Cache a track's raw JSON so `compute_outcomes` can resolve it later + /// without spending a second API call. The quota makes this worth doing + /// eagerly: a search page primes every track it returned. + fn remember(&self, raw: &Value) { + let (Some(cache), Some(id)) = (&self.cache, raw.get("id").and_then(Value::as_str)) else { + return; + }; + if let Ok(json) = serde_json::to_string(raw) { + let _ = cache.put_jamendo_track(id, &json); + } + } + + /// The raw track object for `id`, from the cache when possible. + async fn track_json(&self, id: &str) -> Option { + if let Some(cache) = &self.cache { + if let Ok(Some(hit)) = cache.get_jamendo_track(id) { + if let Ok(v) = serde_json::from_str::(&hit) { + return Some(v); + } + } + } + let client_id = self.client_id.as_ref()?; + let enc = meta_feeder_sdk::common::percent_encode; + let url = format!( + "{}/tracks/?client_id={}&format=json&id={}&include={}&audioformat={}", + self.api_base, + enc(client_id), + enc(id), + enc("licenses musicinfo"), + JAMENDO_AUDIO_FORMAT, + ); + // Retrying matters more here than anywhere: an empty page would + // otherwise make `compute_outcomes` declare a perfectly good track + // permanently unresolvable. + let raw = self.get_results(&url).await.into_iter().next()?; + self.remember(&raw); + Some(raw) + } + + /// Project one track to a wire record. + fn track_record( + &self, + t: &JamendoTrack, + query_filters: &BTreeMap>, + ) -> DiscoveryRecord { + let mut fields: BTreeMap = BTreeMap::new(); + fields.insert("fileType".to_string(), "audio".to_string()); + fields.insert("contentKind".to_string(), "track".to_string()); + fields.insert("title".to_string(), t.name.clone()); + if !t.album_name.is_empty() { + fields.insert("album".to_string(), t.album_name.clone()); + } + if !t.artist_name.is_empty() { + fields.insert("artist".to_string(), t.artist_name.clone()); + fields.insert("albumArtist".to_string(), t.artist_name.clone()); + fields.insert(format!("artists/{}", t.artist_name), "true".to_string()); + } + if let Some(n) = t.position { + fields.insert("trackNumber".to_string(), n.to_string()); + } + // `METADATA_KEYS.md` is explicit that a music tier must NOT invent its + // own duration field: `fileinfo/duration` is the one place a duration + // in seconds lives, whoever produced it. + if let Some(d) = t.duration_secs { + fields.insert("fileinfo/duration".to_string(), d.to_string()); + } + if let Some(d) = t + .releasedate + .as_deref() + .and_then(meta_feeder_sdk::common::normalise_date) + { + fields.insert("releasedate".to_string(), d); + } + fields.insert("licence".to_string(), t.licence.clone()); + // Every byte this tier serves is the `mp32` derivative. + fields.insert("extension".to_string(), "mp3".to_string()); + + // Genres as a KEY-SET, written as Jamendo slugs them (`hiphop`, not + // `Hip Hop`). `record_matches` folds both sides before comparing, so a + // `genres:Pop` filter still matches `genres/pop` — inventing a + // prettier spelling would only invent a second vocabulary. + for g in &t.genres { + fields.insert(format!("genres/{g}"), "true".to_string()); + } + if let Some(lang) = &t.lang { + let l = meta_feeder_sdk::lang::normalize_lang_code(lang); + if !l.is_empty() { + fields.insert(format!("languages/{l}"), "true".to_string()); + } + } + + // The **url locator** (`0x1006`) — what makes this track playable + // straight from a search hit. The real content cid only exists after + // `compute_outcomes` has downloaded the bytes; pre-fetching every track + // in a row to get one would mean downloading twenty songs to render a + // strip. meta-share resolves this lazily on first access, then the real + // digest outranks it. + if let Some(url_cid) = meta_feeder_sdk::hash::compute_url_cid(&t.download_url) { + fields.insert(format!("cids/{url_cid}"), "true".to_string()); + } + + // ⚠ Load-bearing for the whole tier: the consumer's quality gate hides + // anything without a cover, so a track with no artwork never reaches a + // curated row at all. + if let Some(cover) = t.cover_url.as_deref().and_then(meta_feeder_sdk::hash::artwork_locator) { + fields.insert("cover".to_string(), cover); + } + + // Clause 4.1 of the API terms: credit the artist (above), credit + // Jamendo as the provider, and link each item back to its page on the + // platform. `sourceUrl` is the sanctioned attribution field + // (`METADATA_KEYS.md` §5) — click-through metadata, not a byte + // reference, so it stays a URL rather than becoming a cid. + if let Some(share) = &t.share_url { + fields.insert("sourceUrl".to_string(), share.clone()); + } + + fields.insert("jamendoid".to_string(), t.id.clone()); + if !t.album_id.is_empty() { + fields.insert("jamendoalbumid".to_string(), t.album_id.clone()); + } + fields.insert("source/gateway:jamendo".to_string(), "true".to_string()); + + echo_filters(&mut fields, query_filters); + DiscoveryRecord { + upstream_id: "jamendo".to_string(), + record_id: t.id.clone(), + fields, + } + } +} + +#[async_trait] +impl FeederPlugin for JamendoPlugin { + fn upstream_id(&self) -> &'static str { + "jamendo" + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["audio"] + } + + /// **`track` only, deliberately — no `pack`.** + /// + /// The Internet Archive tier serves packs because an archive *item* is a + /// self-contained album directory: listing it gives the release and its + /// complete file set in one call. Jamendo's `/tracks/` page is a slice + /// *across* albums, so a pack synthesised from it would carry a + /// `trackCount` describing how many of that album's tracks happened to land + /// on this page — a number that is wrong in a way no consumer could detect. + /// + /// `accepts_axis` intersects, so a `contentKind:track,pack` query (which is + /// what meta-listen actually sends) still routes here and gets its tracks. + /// A real pack tier wants Jamendo's `/albums/tracks/` endpoint and its own + /// projection. + fn served_content_kinds(&self) -> &'static [&'static str] { + &["track"] + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + self.load_config(cache_dir); + + self.monthly_budget = self + .config + .monthly_request_budget + .filter(|n| *n > 0) + .unwrap_or(JAMENDO_DEFAULT_MONTHLY_BUDGET) + .min(JAMENDO_MONTHLY_QUOTA); + + self.cache = Some( + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin: "jamendo", + source: anyhow::anyhow!("open music cache: {e}"), + })?, + ); + self.hash_cache = Some(meta_feeder_sdk::common::open_midhash_cache( + cache_dir, "jamendo", + )?); + + // Soft-skip rather than fail: the other upstreams in this binary need + // no credential, and a music feeder that refuses to start because one + // optional tier is unconfigured would take the whole free board down. + let Some(id) = self.config.client_id.clone().filter(|s| !s.trim().is_empty()) else { + return Err(ConfigError::MissingConfig { + plugin: "jamendo", + what: "client_id (set it in the dashboard, in the feeder's \ + config.json, or via JAMENDO_CLIENT_ID; register a free \ + app at devportal.jamendo.com)", + }); + }; + self.client_id = Some(id); + Ok(()) + } + + fn health(&self) -> PluginHealth { + let Some(cache) = &self.cache else { + return PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + }; + }; + if self.client_id.is_none() { + return PluginHealth::Degraded { + reason: "no client_id configured".to_string(), + }; + } + let month = utc_year_month(SystemTime::now()); + match cache.jamendo_quota_used(&month) { + Ok(spent) if spent >= self.monthly_budget => PluginHealth::Degraded { + reason: format!( + "monthly request budget exhausted ({spent}/{} for {month})", + self.monthly_budget + ), + }, + _ => PluginHealth::Ok, + } + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + if is_identity_anchored(query) { + debug!( + target: "meta-music", + filters = ?query.filters.keys().collect::>(), + "jamendo: id-anchored query it cannot resolve; returning nothing" + ); + return Ok(Vec::new()); + } + if !wants_content_kind(query, "track") { + return Ok(Vec::new()); + } + + // Over-fetch: an unknown fraction of any page fails the licence / + // download gate, and unlike a rate-limited upstream a second page here + // costs another slice of the *month*. Asking for more up front is + // cheaper than paging. + let want = max_results.clamp(1, JAMENDO_PAGE_MAX); + let limit = (want * 2).min(JAMENDO_PAGE_MAX); + + let Some(url) = self.tracks_url(query, limit) else { + return Ok(Vec::new()); + }; + let mut out = Vec::with_capacity(want); + let mut gated = 0usize; + for raw in self.get_results(&url).await { + let Some(track) = parse_track(&raw) else { + gated += 1; + continue; + }; + self.remember(&raw); + out.push(self.track_record(&track, &query.filters)); + if out.len() >= want { + break; + } + } + if gated > 0 { + debug!( + target: "meta-music", gated, kept = out.len(), + "jamendo: tracks dropped by the licence / download gate" + ); + } + Ok(out) + } + + /// Download one track and content-address it. + /// + /// Full-store branch: real bytes, a real sha2-256 IPFS cid, seeded to + /// bitswap by the gateway core. + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + if let Some(cache) = &self.hash_cache { + if let Some(hit) = meta_feeder_sdk::common::cached_outcome(cache, record_id, "jamendo")? + { + return Ok(hit); + } + } + + let Some(raw) = self.track_json(record_id).await else { + return Err(GatewayError::Permanent(format!( + "jamendo track {record_id} is unresolvable" + ))); + }; + + // ⚠ **Re-check the gate here.** `compute_outcomes` is reachable + // directly by cid, without any prior query, so the check performed + // while projecting a search hit guarantees nothing about this call. A + // track whose artist has since switched free download off, or whose + // licence no longer resolves, must not be fetched and seeded now. + let Some(track) = parse_track(&raw) else { + return Err(GatewayError::Permanent(format!( + "jamendo track {record_id} is not redistributable (no resolvable CC \ + licence, or the artist has not enabled free download)" + ))); + }; + + if self + .budget + .acquire(Duration::from_secs(JAMENDO_WAIT_DEADLINE_SECS)) + .await + == Lease::DeadlineExceeded + { + return Err(GatewayError::RateLimited { retry_after_s: 5 }); + } + // NOTE: no `spend_quota` here. The byte host is + // `prod-*.storage.jamendo.com`, not the `api.jamendo.com` endpoint the + // 35,000/month quota meters, and conflating the two would burn the + // month on downloads. + let resp = self + .http + .get(&track.download_url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("jamendo fetch: {e}")))?; + if !resp.status().is_success() { + return Err(GatewayError::Permanent(format!( + "jamendo fetch {}: HTTP {}", + track.download_url, + resp.status() + ))); + } + // Refuse an oversized body *before* buffering it: the gateway core + // holds the whole outcome in memory to hash and seed it. + if let Some(len) = resp.content_length() { + if len > JAMENDO_MAX_TRACK_BYTES { + return Err(GatewayError::Permanent(format!( + "jamendo track {record_id} is {len} bytes, over the \ + {JAMENDO_MAX_TRACK_BYTES}-byte per-track ceiling" + ))); + } + } + let bytes = resp + .bytes() + .await + .map_err(|e| GatewayError::Transient(format!("jamendo body: {e}")))?; + if bytes.len() as u64 > JAMENDO_MAX_TRACK_BYTES { + return Err(GatewayError::Permanent(format!( + "jamendo track {record_id} exceeded the per-track ceiling after download" + ))); + } + + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + if let Some(cache) = &self.hash_cache { + meta_feeder_sdk::common::store_midhash(cache, record_id, "jamendo", &cid); + } + + Ok(vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record: Some(self.track_record(&track, &BTreeMap::new())), + file_extension: Some("mp3".to_string()), + }]) + } + + async fn handle_fetch(&self, record_id: &str) -> Result, GatewayError> { + let Some(raw) = self.track_json(record_id).await else { + return Ok(None); + }; + // Same gate as `compute_outcomes`, same reason. + let Some(track) = parse_track(&raw) else { + return Ok(None); + }; + let resp = self + .http + .get(&track.download_url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("jamendo fetch: {e}")))?; + if !resp.status().is_success() { + return Ok(None); + } + let stream = futures::StreamExt::map(resp.bytes_stream(), |r| { + r.map_err(|e| GatewayError::Transient(format!("jamendo stream: {e}"))) + }); + Ok(Some(Box::pin(stream))) + } + + fn config_schema(&self) -> ConfigSchema { + ConfigSchema { + fields: vec![ + F::secret("client_id", "Jamendo client ID") + .with_help( + "Free, from devportal.jamendo.com. Without it this tier \ + soft-skips at startup and the rest of the music feeder \ + keeps working. The client SECRET is not needed — the \ + read API authenticates with the ID alone.", + ) + .required(), + F::number("monthly_request_budget", "Monthly API request budget").with_help( + "Jamendo's non-commercial plan allows 35,000 API requests per \ + MONTH — a budget, not a rate: once spent, the tier is dark \ + until the calendar turns. Blank keeps the built-in 30,000, \ + whose headroom covers a second gateway sharing this same \ + client ID (the ledger is per feeder, the quota is per \ + credential).", + ), + ], + } + } + + fn config_values(&self) -> serde_json::Value { + serde_json::json!({ + "client_id": self.config.client_id, + "monthly_request_budget": self.config.monthly_request_budget, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// A verbatim slice of a live `/tracks/` result (track 2180016), trimmed of + /// the `waveform` blob. Every field name here was observed on the wire, not + /// taken from the documentation. + fn live_track() -> Value { + json!({ + "id": "2180016", + "name": "Love You Anymore (feat. Seven)", + "duration": 156, + "artist_id": "505138", + "artist_name": "Tom Orlando", + "album_name": "Love You Anymore (feat. Seven)", + "album_id": "570388", + "license_ccurl": "http://creativecommons.org/licenses/by-sa/3.0/", + "position": 1, + "releasedate": "2024-06-03", + "album_image": "https://usercontent.jamendo.com?type=album&id=570388&width=300&trackid=2180016", + "audio": "https://prod-1.storage.jamendo.com/?trackid=2180016&format=mp32&from=OwGIVd2NxluqXDd%2BO6gTUA%3D%3D", + "audiodownload": "https://prod-1.storage.jamendo.com/download/track/2180016/mp32/", + "shareurl": "https://www.jamendo.com/track/2180016", + "image": "https://usercontent.jamendo.com?type=album&id=570388&width=300&trackid=2180016", + "musicinfo": { + "lang": "en", + "tags": { "genres": ["pop", "hiphop"], "instruments": ["piano"] } + }, + "licenses": { "ccnc": "false", "ccnd": "false", "ccsa": "true" }, + "audiodownload_allowed": true + }) + } + + // -- the month key ------------------------------------------------------ + + #[test] + fn month_key_is_utc_and_handles_boundaries() { + let at = |s: u64| utc_year_month(UNIX_EPOCH + Duration::from_secs(s)); + assert_eq!(at(0), "1970-01"); + // 2026-08-21T00:00:00Z + assert_eq!(at(1_787_270_400), "2026-08"); + // 2024-02-29T12:00:00Z — a leap day, the case the civil-from-days + // shift exists to make unremarkable. + assert_eq!(at(1_709_208_000), "2024-02"); + // One second before and after a year boundary (2025-12-31T23:59:59Z / + // 2026-01-01T00:00:00Z). A month key that slips here would credit + // January's spend to December and hand the tier a double budget. + assert_eq!(at(1_767_225_599), "2025-12"); + assert_eq!(at(1_767_225_600), "2026-01"); + } + + // -- the redistribution gate ------------------------------------------- + + #[test] + fn a_cc_licence_and_an_enabled_download_are_both_required() { + let ok = live_track(); + assert_eq!(redistributable(&ok).as_deref(), Some("CC-BY-SA-3.0")); + + // The artist has not switched free download on. The CC licence still + // permits redistribution in the abstract; the upload does not offer + // the file, so this tier does not take it. + let mut no_dl = live_track(); + no_dl["audiodownload_allowed"] = json!(false); + assert_eq!(redistributable(&no_dl), None); + + // No licence at all ⇒ no record. `METADATA_KEYS.md` §licence: absence + // means "unknown", never "unrestricted". + let mut no_lic = live_track(); + no_lic.as_object_mut().unwrap().remove("license_ccurl"); + assert_eq!(redistributable(&no_lic), None); + + // A URL that is not a recognisable CC grant must not be waved through + // as though it were one. + let mut junk = live_track(); + junk["license_ccurl"] = json!("https://example.com/our-own-terms"); + assert_eq!(redistributable(&junk), None); + } + + /// ⚠ The single most damaging thing this file could get wrong. + /// + /// `audio` carries an expiring `from=` token. A locator cid minted from it + /// is unresolvable within the hour, and since a cid is a hash of the URL + /// string, every query for the same track would mint a *different* cid. + #[test] + fn the_byte_url_is_never_the_signed_streaming_url() { + let t = parse_track(&live_track()).expect("gate passes"); + assert_eq!( + t.download_url, + "https://prod-1.storage.jamendo.com/download/track/2180016/mp32/" + ); + assert!(!t.download_url.contains("from=")); + + // A payload whose only byte URL is a signed one yields no record at + // all, rather than a record with a cid that rots. + let mut signed = live_track(); + signed["audiodownload"] = + json!("https://prod-1.storage.jamendo.com/?trackid=2180016&from=TOKEN%3D%3D"); + assert!(parse_track(&signed).is_none()); + } + + #[test] + fn position_and_duration_survive_both_json_shapes() { + let mut as_strings = live_track(); + as_strings["position"] = json!("7"); + as_strings["duration"] = json!("241"); + let t = parse_track(&as_strings).expect("gate passes"); + assert_eq!(t.position, Some(7)); + assert_eq!(t.duration_secs, Some(241)); + + let t = parse_track(&live_track()).expect("gate passes"); + assert_eq!(t.position, Some(1)); + assert_eq!(t.duration_secs, Some(156)); + } + + // -- the record projection --------------------------------------------- + + #[test] + fn the_record_carries_every_field_the_consumer_gates_on() { + let p = JamendoPlugin::with_base("http://unused".to_string()); + let t = parse_track(&live_track()).expect("gate passes"); + let r = p.track_record(&t, &BTreeMap::new()); + + assert_eq!(r.upstream_id, "jamendo"); + assert_eq!(r.record_id, "2180016"); + let f = |k: &str| r.fields.get(k).map(String::as_str); + + assert_eq!(f("fileType"), Some("audio")); + assert_eq!(f("contentKind"), Some("track")); + assert_eq!(f("title"), Some("Love You Anymore (feat. Seven)")); + assert_eq!(f("artist"), Some("Tom Orlando")); + assert_eq!(f("albumArtist"), Some("Tom Orlando")); + assert_eq!(f("artists/Tom Orlando"), Some("true")); + assert_eq!(f("trackNumber"), Some("1")); + assert_eq!(f("releasedate"), Some("2024-06-03")); + assert_eq!(f("extension"), Some("mp3")); + + // The canonical `id` field every feeder record must carry. + assert_eq!(f("jamendoid"), Some("2180016")); + assert_eq!(f("source/gateway:jamendo"), Some("true")); + + // The licence is an identifier, never the URL it came from. + assert_eq!(f("licence"), Some("CC-BY-SA-3.0")); + assert!(!r.fields["licence"].contains("http")); + + // Duration goes in the ONE registry field for it — a music tier must + // not invent its own. + assert_eq!(f("fileinfo/duration"), Some("156")); + assert!(!r.fields.contains_key("duration")); + + // Clause 4.1 attribution: the backlink to the track's Jamendo page. + assert_eq!(f("sourceUrl"), Some("https://www.jamendo.com/track/2180016")); + + // Genres as a key-set, verbatim as Jamendo slugs them. + assert_eq!(f("genres/pop"), Some("true")); + assert_eq!(f("genres/hiphop"), Some("true")); + assert!(!r.fields.contains_key("genres"), "no legacy csv-set writer"); + + // `en` folds onto the ISO 639-2/B vocabulary the stack compares against. + assert_eq!(f("languages/eng"), Some("true")); + + // Playable from the search hit: a url-locator cid, and a cover, both + // cid-shaped rather than raw URLs. + assert!( + r.fields.keys().any(|k| k.starts_with("cids/")), + "a track with no cid member cannot be played from a search hit" + ); + assert!(f("cover").is_some_and(|c| !c.starts_with("http"))); + } + + /// The gate is re-applied on every path, so a record is never projected + /// for a track this feeder may not host. + #[test] + fn a_gated_track_never_reaches_a_record() { + let mut blocked = live_track(); + blocked["audiodownload_allowed"] = json!(false); + assert!(parse_track(&blocked).is_none()); + } + + // -- routing ------------------------------------------------------------ + + fn query(text: &str, filters: &[(&str, &[&str])]) -> GatewayQuery { + GatewayQuery { + raw_text: text.to_string(), + free_text: text.to_string(), + filters: filters + .iter() + .map(|(k, vs)| { + ( + (*k).to_string(), + vs.iter().map(|v| (*v).to_string()).collect(), + ) + }) + .collect(), + ranges: Vec::new(), + negations: Vec::new(), + } + } + + #[test] + fn a_keywordless_query_asks_for_the_popularity_row() { + let mut p = JamendoPlugin::with_base("http://api.test".to_string()); + p.client_id = Some("abc123".to_string()); + let url = p + .tracks_url(&query("", &[("fileType", &["audio"])]), 20) + .expect("client id present"); + assert!(url.contains(&format!("order={JAMENDO_BROWSE_ORDER}"))); + assert!(!url.contains("search=")); + assert!(url.contains("client_id=abc123")); + } + + #[test] + fn free_text_uses_search_not_namesearch() { + let mut p = JamendoPlugin::with_base("http://api.test".to_string()); + p.client_id = Some("abc123".to_string()); + let url = p.tracks_url(&query("miles davis", &[]), 10).expect("url"); + // `search` spans artist and album too; `namesearch` only matches the + // track title, so every artist query would come back empty. + assert!(url.contains("search=miles%20davis") || url.contains("search=miles+davis")); + assert!(!url.contains("namesearch=")); + } + + /// A genre-scoped row must ask the upstream for that genre. Fetching a + /// popularity page and letting `record_matches` drop everything is how a + /// genre row ends up permanently empty. + #[test] + fn a_genre_filter_is_pushed_down_to_the_upstream() { + let mut p = JamendoPlugin::with_base("http://api.test".to_string()); + p.client_id = Some("abc123".to_string()); + let url = p + .tracks_url(&query("", &[("genres", &["Jazz"])]), 10) + .expect("url"); + assert!(url.contains("tags=jazz")); + } + + /// Without a credential the plugin builds no URL at all, rather than + /// calling an endpoint that answers `HTTP 200` with an error envelope. + #[test] + fn no_client_id_means_no_request() { + let p = JamendoPlugin::with_base("http://api.test".to_string()); + assert!(p.tracks_url(&query("anything", &[]), 10).is_none()); + } + + #[tokio::test] + async fn an_id_anchored_query_returns_nothing() { + let p = JamendoPlugin::with_base("http://api.test".to_string()); + let q = query( + "", + &[ + ("fileType", &["audio"]), + ("contentKind", &["track"]), + ("mbReleaseGroupId", &["f5093c06-23e3-404f-aeaa-40f72885ee3a"]), + ], + ); + // Jamendo holds no MusicBrainz ids. Answering an album page with a + // popularity browse is how a BTS album page fills with unrelated music. + assert!(p.handle_query(&q, 20).await.expect("ok").is_empty()); + } + + #[tokio::test] + async fn a_card_query_is_not_this_plugins_business() { + let p = JamendoPlugin::with_base("http://api.test".to_string()); + let q = query("", &[("fileType", &["card"]), ("contentKind", &["album"])]); + assert!(p.handle_query(&q, 20).await.expect("ok").is_empty()); + } + + // -- the quota ledger --------------------------------------------------- + + /// The month is a budget, not a rate: it does not refill, so the ledger + /// must refuse the call *before* it is made. + #[test] + fn the_monthly_budget_is_spent_once_and_then_refuses() { + let dir = tempfile::tempdir().expect("tempdir"); + let cache = MidhashCache::open(dir.path()).expect("open"); + + assert_eq!(cache.jamendo_quota_spend("2026-08", 3).unwrap(), Some(1)); + assert_eq!(cache.jamendo_quota_spend("2026-08", 3).unwrap(), Some(2)); + assert_eq!(cache.jamendo_quota_spend("2026-08", 3).unwrap(), Some(3)); + // Budget reached — every further call is refused. + assert_eq!(cache.jamendo_quota_spend("2026-08", 3).unwrap(), None); + assert_eq!(cache.jamendo_quota_used("2026-08").unwrap(), 3); + + // The next month starts clean, which is the whole reason the ledger is + // keyed by month rather than being a single counter. + assert_eq!(cache.jamendo_quota_spend("2026-09", 3).unwrap(), Some(1)); + assert_eq!(cache.jamendo_quota_used("2026-08").unwrap(), 3); + } + + /// An unmetered upstream with a hard monthly ceiling is worse than an + /// absent one, so a plugin with no ledger open refuses rather than calls. + #[test] + fn no_ledger_means_no_spending() { + let p = JamendoPlugin::with_base("http://api.test".to_string()); + assert!(!p.spend_quota()); + } + + // -- the error envelope ------------------------------------------------- + + /// ⚠ Jamendo answers a dead credential with **HTTP 200** and an error in + /// the body. A client that trusts the status code turns an expired + /// client_id into a silently empty tier. + #[tokio::test] + async fn a_200_with_an_error_envelope_is_treated_as_a_failure() { + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "headers": { + "status": "failed", + "code": 5, + "error_message": "Jamendo Api Invalid Client Id Error: Your credential is not authorized.", + "results_count": 0 + }, + "results": [] + }))) + .mount(&server) + .await; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut p = JamendoPlugin::with_base(server.uri()); + p.client_id = Some("dead-key".to_string()); + p.cache = Some(MidhashCache::open(dir.path()).expect("open")); + + assert!(p.get_json(&format!("{}/tracks/", server.uri())).await.is_none()); + } + + /// ⚠ The upstream answers a good query with an empty success often enough + /// (up to 4-in-8 on a browse row, measured) that a single-attempt client + /// blanks rows at random. Only an *empty success* is retried — an error + /// envelope must fail fast, or a dead credential triples the quota burn. + #[tokio::test] + async fn an_empty_success_is_retried_but_an_error_envelope_is_not() { + use wiremock::matchers::method; + + let server = wiremock::MockServer::start().await; + // The first two calls come back empty-but-successful, exactly as the + // live API does; the third carries the page. + wiremock::Mock::given(method("GET")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "headers": { "status": "success", "code": 0, "results_count": 0 }, + "results": [] + }))) + .up_to_n_times(2) + .with_priority(1) + .mount(&server) + .await; + wiremock::Mock::given(method("GET")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "headers": { "status": "success", "code": 0, "results_count": 1 }, + "results": [live_track()] + }))) + .with_priority(2) + .mount(&server) + .await; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut p = JamendoPlugin::with_base(server.uri()); + p.client_id = Some("abc123".to_string()); + p.cache = Some(MidhashCache::open(dir.path()).expect("open")); + + let q = query("love", &[("fileType", &["audio"]), ("contentKind", &["track"])]); + let records = p.handle_query(&q, 10).await.expect("ok"); + assert_eq!(records.len(), 1, "the retry must rescue the page"); + + // An error envelope is NOT retried: one call in, one call out. + let failing = wiremock::MockServer::start().await; + wiremock::Mock::given(method("GET")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "headers": { "status": "failed", "code": 5, "error_message": "bad key" }, + "results": [] + }))) + .expect(1) + .mount(&failing) + .await; + let dir2 = tempfile::tempdir().expect("tempdir"); + let mut p2 = JamendoPlugin::with_base(failing.uri()); + p2.client_id = Some("dead".to_string()); + p2.cache = Some(MidhashCache::open(dir2.path()).expect("open")); + assert!(p2.handle_query(&q, 10).await.expect("ok").is_empty()); + // `expect(1)` is asserted when the server drops. + } + + /// **Live** smoke test against the real API. Ignored by default so `cargo + /// test` stays hermetic and spends nobody's monthly quota; run it + /// deliberately after touching the request shape or the parser: + /// + /// ```text + /// JAMENDO_CLIENT_ID=... cargo test -p music-feeder --lib -- --ignored --nocapture + /// ``` + /// + /// It asserts the things a wiremock fixture structurally cannot: that the + /// parameters this plugin sends are still the ones Jamendo answers, and + /// that a real page survives the licence / download gate. + #[tokio::test] + #[ignore = "hits the live Jamendo API; needs JAMENDO_CLIENT_ID"] + async fn live_smoke() { + let Ok(client_id) = std::env::var("JAMENDO_CLIENT_ID") else { + panic!("set JAMENDO_CLIENT_ID to run the live smoke test"); + }; + let dir = tempfile::tempdir().expect("tempdir"); + let mut p = JamendoPlugin::new(); + p.client_id = Some(client_id); + p.cache = Some(MidhashCache::open(dir.path()).expect("open")); + + for (label, q) in [ + ( + "browse row", + query("", &[("fileType", &["audio"]), ("contentKind", &["track"])]), + ), + ( + "free text", + query("love", &[("fileType", &["audio"]), ("contentKind", &["track"])]), + ), + ( + "genre row", + query("", &[("fileType", &["audio"]), ("genres", &["jazz"])]), + ), + ] { + let records = p.handle_query(&q, 10).await.expect("query ok"); + println!("[{label}] {} records", records.len()); + assert!(!records.is_empty(), "{label} returned nothing"); + for r in &records { + let f = |k: &str| r.fields.get(k).cloned().unwrap_or_default(); + assert!(!f("licence").is_empty(), "{label}: record with no licence"); + assert!( + r.fields.keys().any(|k| k.starts_with("cids/")), + "{label}: record with no playable cid" + ); + assert!(!f("cover").is_empty(), "{label}: record with no cover"); + assert!( + f("sourceUrl").starts_with("https://www.jamendo.com/"), + "{label}: missing the clause 4.1 backlink" + ); + } + let first = &records[0]; + println!( + " e.g. {} — {} [{}] {}", + first.fields.get("artist").cloned().unwrap_or_default(), + first.fields.get("title").cloned().unwrap_or_default(), + first.fields.get("licence").cloned().unwrap_or_default(), + first.fields.get("sourceUrl").cloned().unwrap_or_default(), + ); + } + } + + /// End to end through the real router: a wiremock page in, gated records + /// out. + #[tokio::test] + async fn a_search_page_becomes_records_with_the_gated_rows_dropped() { + let mut blocked = live_track(); + blocked["id"] = json!("999"); + blocked["audiodownload_allowed"] = json!(false); + + let server = wiremock::MockServer::start().await; + wiremock::Mock::given(wiremock::matchers::method("GET")) + .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(json!({ + "headers": { "status": "success", "code": 0, "results_count": 2 }, + "results": [live_track(), blocked] + }))) + .mount(&server) + .await; + + let dir = tempfile::tempdir().expect("tempdir"); + let mut p = JamendoPlugin::with_base(server.uri()); + p.client_id = Some("abc123".to_string()); + p.cache = Some(MidhashCache::open(dir.path()).expect("open")); + + let q = query("love", &[("fileType", &["audio"]), ("contentKind", &["track"])]); + let records = p.handle_query(&q, 20).await.expect("ok"); + + assert_eq!(records.len(), 1, "the download-disabled row must be dropped"); + assert_eq!(records[0].record_id, "2180016"); + assert_eq!( + records[0].fields.get("licence").map(String::as_str), + Some("CC-BY-SA-3.0") + ); + + // The page primed the per-track cache, so resolving this record later + // costs no second API call — the quota makes that worth doing eagerly. + assert!(p + .cache + .as_ref() + .unwrap() + .get_jamendo_track("2180016") + .unwrap() + .is_some()); + } +} diff --git a/feeder-plugin/jamendo-feeder/src/lib.rs b/feeder-plugin/jamendo-feeder/src/lib.rs new file mode 100644 index 0000000..3653244 --- /dev/null +++ b/feeder-plugin/jamendo-feeder/src/lib.rs @@ -0,0 +1,7 @@ +//! `jamendo-feeder` library surface. +//! +//! Jamendo Creative-Commons music. Opt-in: needs a client id. + +pub mod consts; +pub mod jamendo; +pub mod shared; diff --git a/feeder-plugin/jamendo-feeder/src/main.rs b/feeder-plugin/jamendo-feeder/src/main.rs new file mode 100644 index 0000000..5051b5e --- /dev/null +++ b/feeder-plugin/jamendo-feeder/src/main.rs @@ -0,0 +1,33 @@ +//! `jamendo-feeder` — a single-upstream feeder sidecar. +//! +//! Jamendo Creative-Commons music. Opt-in: needs a client id. +//! +//! Upstream: Jamendo API v3.0 (api.jamendo.com) +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use meta_feeder_sdk::plugin::FeederPlugin; +use meta_feeder_sdk::serve_feeders; +use jamendo_feeder::jamendo::JamendoPlugin; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(JamendoPlugin::new())]; + serve_feeders(plugins, state_dir, listen).await +} diff --git a/feeder-plugin/jamendo-feeder/src/shared.rs b/feeder-plugin/jamendo-feeder/src/shared.rs new file mode 100644 index 0000000..79c6a1b --- /dev/null +++ b/feeder-plugin/jamendo-feeder/src/shared.rs @@ -0,0 +1,150 @@ +//! Helpers shared by every **content-tier** upstream in this crate. +//! +//! `internetarchive` and `jamendo` are independent bridges to unrelated +//! services, but they answer the same shape of question — "which playable +//! files match this query?" — and so they need the same three pieces of +//! plumbing. These began as private items in `internetarchive.rs`; they are +//! here rather than duplicated so that a fix to the filter-echo rule (a subtle +//! one — see below) cannot land in one upstream and be forgotten in the other. +//! +//! ⚠ `internetarchive.rs` still carries its own private copies of +//! `echo_filters` / `wants_content_kind` and its own inline identity-filter +//! list. Collapsing them onto this module is a mechanical follow-up, deferred +//! only because that file had unrelated edits in flight when this landed. +//! +//! The card tier deliberately does NOT use these: a card is a *work*, not a +//! file, and its projection has its own filter rules in `card.rs`. + +use std::collections::BTreeMap; + +use meta_feeder_sdk::plugin::GatewayQuery; + +/// Echo back structured filters the record did not already set, so the record +/// survives `record_matches` at the gateway and again at meta-search. +/// +/// The two exclusions are not arbitrary: +/// +/// - **`languages`** and **`genres`** are `key-set` fields +/// (`genres/ = "true"`), and `record_matches` has a dedicated arm for +/// each. Echoing a flat `genres = "jazz,pop"` value would write the legacy +/// csv-set shape that `METADATA_KEYS.md` §14.12 forbids for new writers, and +/// it would not help the record match anyway. +/// - A key the record **already set** is never overwritten. The record's own +/// value is the truth; the filter is only what was asked for. +pub(crate) fn echo_filters( + fields: &mut BTreeMap, + query_filters: &BTreeMap>, +) { + for (key, allowed) in query_filters { + if key == "languages" || key == "genres" || fields.contains_key(key) || allowed.is_empty() { + continue; + } + fields.insert(key.clone(), allowed.join(",")); + } +} + +/// Does the query ask for this content kind? No filter means everything the +/// calling plugin serves. +pub(crate) fn wants_content_kind(query: &GatewayQuery, kind: &str) -> bool { + match query.filters.get("contentKind") { + None => true, + Some(values) if values.is_empty() => true, + Some(values) => values.iter().any(|v| v.trim().eq_ignore_ascii_case(kind)), + } +} + +/// Filters that name a *specific work* by id. +/// +/// Neither content upstream holds MusicBrainz identifiers, so a query carrying +/// one is a question they cannot answer. +const IDENTITY_FILTERS: &[&str] = &[ + "mbReleaseGroupId", + "mbReleaseId", + "mbArtistId", + "mbRecordingId", + "isrc", + "tmdbid", + "imdbid", +]; + +/// ⚠ **An identity-anchored query a content upstream cannot satisfy must +/// return NOTHING**, not a free-text browse. +/// +/// A query like `mbReleaseGroupId: fileType:audio` carries no free text +/// for a keyword upstream to search on. Falling through to the keyword path +/// then means an *empty* search — which both upstreams happily answer with +/// their most-popular items. +/// +/// Observed live on the Internet Archive tier before this guard existed: +/// opening a BTS album filled its page with sixty Grateful Dead and +/// shortwave-numbers-station recordings, every one of them a confident-looking +/// answer to a question nobody asked. A consumer's album page is exactly this +/// query shape, so the failure is not an edge case — it is the normal +/// detail-page request. +pub(crate) fn is_identity_anchored(query: &GatewayQuery) -> bool { + IDENTITY_FILTERS + .iter() + .any(|k| query.filters.contains_key(*k)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn query(filters: &[(&str, &[&str])]) -> GatewayQuery { + let mut q = GatewayQuery::from_free_text(""); + for (k, vs) in filters { + q.filters.insert( + (*k).to_string(), + vs.iter().map(|v| (*v).to_string()).collect(), + ); + } + q + } + + #[test] + fn identity_anchored_queries_are_recognised() { + assert!(is_identity_anchored(&query(&[( + "mbReleaseGroupId", + &["f5093c06"] + )]))); + assert!(is_identity_anchored(&query(&[("isrc", &["USRC17607839"])]))); + // A plain typed query is not anchored — it is the keyword path. + assert!(!is_identity_anchored(&query(&[ + ("fileType", &["audio"]), + ("contentKind", &["track"]) + ]))); + } + + #[test] + fn content_kind_absent_means_everything() { + let q = query(&[("fileType", &["audio"])]); + assert!(wants_content_kind(&q, "track")); + assert!(wants_content_kind(&q, "pack")); + + let q = query(&[("contentKind", &["pack"])]); + assert!(!wants_content_kind(&q, "track")); + assert!(wants_content_kind(&q, "pack")); + } + + /// The key-set exclusions. Echoing a flat `genres` value would write the + /// forbidden csv-set shape and would not make the record match anyway. + #[test] + fn key_set_filters_are_never_echoed_flat() { + let mut fields = BTreeMap::new(); + fields.insert("fileType".to_string(), "audio".to_string()); + let mut filters = BTreeMap::new(); + filters.insert("genres".to_string(), vec!["jazz".to_string()]); + filters.insert("languages".to_string(), vec!["eng".to_string()]); + filters.insert("quality".to_string(), vec!["lossless".to_string()]); + // `fileType` is already set by the record and must not be clobbered. + filters.insert("fileType".to_string(), vec!["video".to_string()]); + + echo_filters(&mut fields, &filters); + + assert!(!fields.contains_key("genres")); + assert!(!fields.contains_key("languages")); + assert_eq!(fields.get("quality").map(String::as_str), Some("lossless")); + assert_eq!(fields.get("fileType").map(String::as_str), Some("audio")); + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]