From 33f31d0d8bde6ade14bac26e3f071d14cb62b6c0 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 24 Aug 2026 19:02:27 +0200 Subject: [PATCH] feeder rework --- .gitattributes | 6 + .github/workflows/publish.yml | 59 + Cargo.lock | 1969 +++++++++++++++++++ Cargo.toml | 9 + README.md | 18 +- feeder-plugin/youtube-feeder/Cargo.toml | 45 + feeder-plugin/youtube-feeder/Dockerfile | 44 + feeder-plugin/youtube-feeder/src/consts.rs | 57 + feeder-plugin/youtube-feeder/src/lib.rs | 6 + feeder-plugin/youtube-feeder/src/main.rs | 33 + feeder-plugin/youtube-feeder/src/youtube.rs | 1480 ++++++++++++++ rust-toolchain.toml | 3 + 12 files changed, 3728 insertions(+), 1 deletion(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/publish.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 feeder-plugin/youtube-feeder/Cargo.toml create mode 100644 feeder-plugin/youtube-feeder/Dockerfile create mode 100644 feeder-plugin/youtube-feeder/src/consts.rs create mode 100644 feeder-plugin/youtube-feeder/src/lib.rs create mode 100644 feeder-plugin/youtube-feeder/src/main.rs create mode 100644 feeder-plugin/youtube-feeder/src/youtube.rs create mode 100644 rust-toolchain.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..cc7ea24 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/paper-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/youtube-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..0050707 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1969 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "musicbrainz-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "meta-feeder-sdk", + "redb", + "reqwest", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "youtube-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "meta-feeder-sdk", + "musicbrainz-feeder", + "redb", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e3e4a80 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-youtube" diff --git a/README.md b/README.md index 7f1fd49..1a3cf09 100644 --- a/README.md +++ b/README.md @@ -1 +1,17 @@ -# meta-feeder-youtube \ No newline at end of file +# meta-feeder-youtube + +YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded. + +**Upstream:** YouTube via Piped instances +**Role:** delegated + +A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin` +from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk). + +```bash +cargo build --release --bin youtube-feeder +cargo test +``` + +Scope and naming conventions: `docs/project-architecture/feeder-architecture.md` +in the meta-root. diff --git a/feeder-plugin/youtube-feeder/Cargo.toml b/feeder-plugin/youtube-feeder/Cargo.toml new file mode 100644 index 0000000..6dbbd5e --- /dev/null +++ b/feeder-plugin/youtube-feeder/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "youtube-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded. +# Upstream: YouTube via Piped instances + +[lib] +name = "youtube_feeder" +path = "src/lib.rs" + +[[bin]] +name = "youtube-feeder" +path = "src/main.rs" + +[dependencies] +# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to: +# meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "" } +meta-feeder-sdk = { path = "../../../meta-feeder-sdk" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +bytes = "1" +futures = "0.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +redb = "2" +base64 = "0.22" +# The MusicBrainz client, from the crate that owns it. Only the *music* +# path uses it; the video path never touches MusicBrainz. +musicbrainz-feeder = { path = "../../../meta-feeder-musicbrainz/feeder-plugin/musicbrainz-feeder" } + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/youtube-feeder/Dockerfile b/feeder-plugin/youtube-feeder/Dockerfile new file mode 100644 index 0000000..b1ae8c6 --- /dev/null +++ b/feeder-plugin/youtube-feeder/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# youtube-feeder sidecar. Multi-stage build; build context is this repo's root. +# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied +# in — `git` must be present in the builder for cargo to fetch it. + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=youtube-feeder-target,target=/build/target \ + cargo build --release --bin youtube-feeder \ + && cp target/release/youtube-feeder /youtube-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /youtube-feeder /usr/local/bin/youtube-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/youtube-feeder"] diff --git a/feeder-plugin/youtube-feeder/src/consts.rs b/feeder-plugin/youtube-feeder/src/consts.rs new file mode 100644 index 0000000..5e51f41 --- /dev/null +++ b/feeder-plugin/youtube-feeder/src/consts.rs @@ -0,0 +1,57 @@ +//! Constants for `youtube-feeder`. + +/// User-Agent for outbound HTTP. +/// +/// ⚠ **Not cosmetic.** Several upstreams reject a generic or absent +/// User-Agent, and a 403 there looks exactly like a rate-limit — easy to +/// misdiagnose. Identify the application and carry contact information. +pub const USER_AGENT: &str = concat!( + "youtube-feeder/", + env!("CARGO_PKG_VERSION"), + " ( https://github.com/worph/meta-feeder-youtube )" +); + +/// Piped instances, tried in order, rotating past a dead one. +/// +/// **A pool, not a hostname — measured.** Of four instances tried during the +/// study, one returned `502`, two `301`-redirected away, and one answered in +/// ~1 s. That is the failure profile to design for. It is cheap to survive +/// because we take only the 11-character video id: a dead instance is a retry, +/// not a broken stream (contrast the *bytes* argument against Piped in +/// `youtube-source-integration-study.md` §2.2, which does not apply here). +pub const PIPED_INSTANCES: &[&str] = &[ + "https://api.piped.private.coffee", + "https://pipedapi.kavin.rocks", + "https://pipedapi.adminforge.de", + "https://api.piped.yt", +]; +/// Per-request timeout against a Piped instance. +pub const PIPED_TIMEOUT_SECS: u64 = 15; +/// Artwork for a YouTube rendition. `hqdefault` exists for every video, unlike +/// `maxresdefault`, which 404s on anything not uploaded in HD — and a 404 here +/// is **silent**, gating the record out with no error anywhere. +pub const YT_THUMB_BASE: &str = "https://i.ytimg.com/vi"; +/// How far a candidate's runtime may sit from the MusicBrainz recording length +/// and still be the same recording. +/// +/// ±10 s is what the study measured at, and it is doing two jobs: absorbing +/// fade-in/out and pre-gap differences between a CD master and an uploaded +/// Art Track, while staying tight enough to separate a radio edit (249 s) from +/// an album cut (370 s). +pub const YT_DURATION_TOLERANCE_SECS: i64 = 10; +/// Ceiling on Piped `music_songs` candidates examined per track. +pub const YT_CANDIDATES_PER_TRACK: usize = 8; +/// How many ranked YouTube references one recording may carry. +/// +/// §3's mitigation for reference rot: the video **id** is immutable, but its +/// *availability* is not — takedowns, geo-blocks and embed-disabled uploads all +/// kill a reference whose CID stays valid forever. One reference per recording +/// means a takedown kills the track; several let the client fall through. +/// +/// Three rather than "all of them" because each is a record replicated across +/// the mesh, and the tail past three is candidates the resolver already ranked +/// as worse matches. +pub const MAX_REFS_PER_RECORDING: usize = 3; + +// MusicBrainz's published rate ceiling, from the crate that owns that client. +pub use musicbrainz_feeder::consts::{MB_BURST, MB_RATE_PER_SEC, MB_TIMEOUT_SECS}; diff --git a/feeder-plugin/youtube-feeder/src/lib.rs b/feeder-plugin/youtube-feeder/src/lib.rs new file mode 100644 index 0000000..d5c0dbf --- /dev/null +++ b/feeder-plugin/youtube-feeder/src/lib.rs @@ -0,0 +1,6 @@ +//! `youtube-feeder` library surface. +//! +//! YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded. + +pub mod consts; +pub mod youtube; diff --git a/feeder-plugin/youtube-feeder/src/main.rs b/feeder-plugin/youtube-feeder/src/main.rs new file mode 100644 index 0000000..21d0146 --- /dev/null +++ b/feeder-plugin/youtube-feeder/src/main.rs @@ -0,0 +1,33 @@ +//! `youtube-feeder` — a single-upstream feeder sidecar. +//! +//! YouTube references as a delegated-playback tier: playback happens on YouTube, no bytes are ever transported or seeded. +//! +//! Upstream: YouTube via Piped instances +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use meta_feeder_sdk::plugin::FeederPlugin; +use meta_feeder_sdk::serve_feeders; +use youtube_feeder::youtube::YouTubePlugin; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(YouTubePlugin::default())]; + serve_feeders(plugins, state_dir, listen).await +} diff --git a/feeder-plugin/youtube-feeder/src/youtube.rs b/feeder-plugin/youtube-feeder/src/youtube.rs new file mode 100644 index 0000000..9a1d61c --- /dev/null +++ b/feeder-plugin/youtube-feeder/src/youtube.rs @@ -0,0 +1,1480 @@ +//! The `youtube` upstream — the **delegated-playback** tier. +//! +//! Resolves a MusicBrainz release-group to per-track YouTube references so a +//! recording with no owned bytes anywhere is still *playable*. It is the third +//! plugin in this binary, beside `musicbrainz` (identity) and +//! `internetarchive` (free bytes), and it lives here rather than in a repo of +//! its own because the MusicBrainz client, the rate budget, the cache and the +//! MBID→tracklist resolution it needs are all already written next door. +//! +//! # What makes this tier different from every other one +//! +//! Every other version in MetaMesh is *bytes at a cid meta-share can serve* — +//! owned already, or fetchable and then seeded. A YouTube reference is a third +//! thing: **playback we can only delegate**. Nothing is transported, seeded or +//! content-addressed, and the byte path REFUSES the cid (`400`) rather than +//! 404ing. See `docs/cid-formats.md` §7 and the delegated-playback study. +//! +//! Consequences that are architecture, not decoration: +//! +//! * The outcome is **metadata-only** — the `(None, Some)` branch, `bytes: +//! None`, no `handle_fetch`, no `get_blob`, no bytes ever. +//! * `HashKind::YtVideo` keeps it out of the blockstore seed, which is gated on +//! `Sha2_256` anyway — but the explicit kind is what lets the core, and any +//! future consumer, tell "no bytes *yet*" from "no bytes, ever". +//! * A reference is an **on-ramp, not a replacement**: because it hangs off the +//! same record as any future real cid, the version list upgrades in place and +//! the tier ordering demotes the reference automatically. +//! +//! # The resolver, and why it refuses more than it emits +//! +//! The gateway path here is well-trodden; the real risk is **match quality**, +//! and the asymmetry that makes it matter: a *missing* track shows up as an +//! honestly sparse album page, while a *wrong* one shows up as a working app +//! playing the wrong song. So this resolver is built to refuse. +//! +//! Measured over 83 tracks / 12 albums, deliberately unkind sample (study §7.1): +//! 72.3% EXACT (channel **and** duration agree), 15.7% TITLE, 3.6% genuine coin +//! flips. **Only EXACT is emitted.** That is the deliberate trade: 72.3% +//! coverage and an honestly sparser page, rather than 88.0% with a confidence +//! marker the UI would have to render. +//! +//! The two filters are both load-bearing and neither is sufficient alone: +//! +//! * **Channel id == the artist's YouTube-Music channel** answers "the right +//! artist, officially" as an exact id comparison rather than a bag of +//! heuristics (`- Topic` suffixes, verified badges, string similarity). +//! * **Duration within ±10 s of the MusicBrainz recording length** answers "the +//! right *recording*". MusicBrainz holds three "Get Lucky" recordings +//! (237/294/40 s) and Piped returned 370 s and 249 s; without duration you +//! cannot tell which one you queued. + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, GatewayQuery, HashKind, HashOutcome}; +use meta_feeder_sdk::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; +use serde_json::Value; +use tracing::{debug, warn}; + +use meta_feeder_sdk::cache::MidhashCache; +use crate::consts::{ + MAX_REFS_PER_RECORDING, MB_BURST, MB_RATE_PER_SEC, MB_TIMEOUT_SECS, PIPED_INSTANCES, + PIPED_TIMEOUT_SECS, USER_AGENT, + YT_CANDIDATES_PER_TRACK, YT_DURATION_TOLERANCE_SECS, YT_THUMB_BASE, +}; +use musicbrainz_feeder::client::{primary_artist, MusicBrainzClient}; +use meta_feeder_sdk::budget::RateBudget; + +/// Which key produced a channel match, in the priority order the resolver +/// tries them. +/// +/// **Recorded on the record**, not just used and dropped. Even emitting EXACT +/// only, the key that produced a match is the difference between "MusicBrainz +/// says this is the artist's official music channel" and "a name match against +/// YouTube's own artist index agreed" — and retrofitting the distinction onto +/// records already replicated across the mesh is painful. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ChannelKey { + /// MusicBrainz `youtube music` relation — authoritative, sparse. + MbYoutubeMusic, + /// MusicBrainz `youtube` relation — the artist's *regular* channel. + /// + /// Weaker, and the weakness is specific: for BTS this is not where YouTube + /// Music serves from, and matching on it scored **0/8** exact. It is not + /// useless either — Nujabes and Boards of Canada both matched 8/8 through + /// it, because artists with no separate YouTube Music artist channel get + /// their regular uploads surfaced by `music_songs`. Second, never first. + MbYoutube, + /// Piped's own `music_artists` index — name-matched, so the weakest key, + /// but it has **no coverage gap**, which is why it exists. + /// + /// MusicBrainz channel coverage looked like a hard ceiling (77% exact where + /// a relation exists, **0%** where it does not — 0% *by construction*, since + /// with no channel id nothing can reach EXACT however obviously right it + /// is). Adding this third key lifted the sample from 53.0% to 72.3% EXACT + /// and cut the risky residue from 4.8% to 3.6%. + PipedMusicArtists, +} + +impl ChannelKey { + /// The value written to the record's `ytMatchKey` field. + pub fn as_str(self) -> &'static str { + match self { + ChannelKey::MbYoutubeMusic => "mb:youtube-music", + ChannelKey::MbYoutube => "mb:youtube", + ChannelKey::PipedMusicArtists => "piped:music-artists", + } + } +} + +/// An artist's candidate channels, best key first. Empty means "no key +/// available", which makes every track for that artist unresolvable — by +/// design, since EXACT requires a channel comparison. +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +pub struct ChannelKeys { + /// `(channel_id, key_label)` pairs in priority order. + pub ranked: Vec<(String, String)>, +} + +/// One Piped `music_songs` result, trimmed to what the filters read. +#[derive(Clone, Debug)] +struct Candidate { + video_id: String, + title: String, + uploader_channel: String, + duration_secs: i64, +} + +/// The verdict for one (track, candidate) pair. +/// +/// Only [`Verdict::Exact`] is ever emitted. The other variants exist so the +/// refusals are *named* — in particular [`Verdict::ChannelWrongCut`], which is +/// the single most dangerous shape in this tier and must never be widened into +/// a near-miss. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Verdict { + /// Channel **and** duration agree. Safe to emit. + Exact, + /// The channel agrees but the runtime does not — a **refusal**, never a + /// near-miss. + /// + /// # The artist-credit trap + /// + /// This is where classical breaks, and it breaks confidently. MusicBrainz + /// credits the **composer**, not the performer: + /// + /// ```text + /// Glenn Gould / The Goldberg Variations -> MB artist-credit = "Johann Sebastian Bach" + /// ``` + /// + /// So the resolver looks up *Bach's* channel, finds *a* Goldberg + /// Variations on it, and the channel key — the strongest signal it has — + /// agrees. The probe scored those at **+32 s and +209 s**: right composer, + /// wrong performance. Widening the tolerance to "rescue" them would emit + /// someone else's recording under this track's name. + ChannelWrongCut, + /// No channel agreement, or nothing plausible at all. + NoMatch, +} + +pub struct YouTubePlugin { + http: reqwest::Client, + cache: Option, + mb: Option>, + /// Rotating cursor into [`PIPED_INSTANCES`] — see [`Self::piped_get`]. + instance: AtomicUsize, + /// Test hook: pin Piped to a wiremock base instead of the pool. + piped_base: Option, + /// Test hook: pin the MusicBrainz base. + mb_base: Option, +} + +impl Default for YouTubePlugin { + fn default() -> Self { + Self { + http: reqwest::Client::builder() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(PIPED_TIMEOUT_SECS)) + .build() + .expect("build youtube http client"), + cache: None, + mb: None, + instance: AtomicUsize::new(0), + piped_base: None, + mb_base: None, + } + } +} + +impl YouTubePlugin { + /// Test constructor: pin both upstreams at wiremock servers. + pub fn with_bases(piped_base: String, mb_base: String) -> Self { + Self { + piped_base: Some(piped_base), + mb_base: Some(mb_base), + ..Default::default() + } + } + + /// One GET against the Piped pool, rotating past a dead instance. + /// + /// The rotation is the whole reliability story for this leg, and it is + /// affordable precisely because we take only the 11-character video id: a + /// dead instance costs a retry, not a broken stream. + async fn piped_get(&self, path_and_query: &str) -> Option { + if let Some(base) = &self.piped_base { + let url = format!("{}/{}", base.trim_end_matches('/'), path_and_query); + return self.http.get(&url).send().await.ok()?.json().await.ok(); + } + let start = self.instance.load(Ordering::Relaxed); + for i in 0..PIPED_INSTANCES.len() { + let idx = (start + i) % PIPED_INSTANCES.len(); + let url = format!("{}/{}", PIPED_INSTANCES[idx], path_and_query); + match self.http.get(&url).send().await { + Ok(r) if r.status().is_success() => match r.json::().await { + Ok(v) => { + // Stick to a working instance for subsequent calls. + self.instance.store(idx, Ordering::Relaxed); + return Some(v); + } + Err(e) => debug!(target: "meta-music", %url, error = %e, "piped: bad json"), + }, + Ok(r) => { + debug!(target: "meta-music", %url, status = %r.status(), "piped: non-2xx, rotating") + } + Err(e) => debug!(target: "meta-music", %url, error = %e, "piped: transport, rotating"), + } + } + warn!(target: "meta-music", path = %path_and_query, "piped: every instance failed"); + None + } + + /// Resolve an artist's channel keys, best first. + /// + /// Cached **positively only**: MusicBrainz answers `503` when crowded and + /// the client reports that the same way it reports an empty result, so + /// caching a negative would freeze a transport hiccup into "this artist has + /// no channel" forever. The probe fabricated exactly that verdict — it + /// reported "no YouTube channel in MusicBrainz" for artists that plainly + /// have one — until the failure path was separated from the empty one. + async fn channel_keys(&self, artist_mbid: &str, artist_name: &str) -> ChannelKeys { + let cache_key = artist_mbid; + if let Some(c) = self.cache.as_ref() { + if let Ok(Some(hit)) = c.get_yt_channel(cache_key) { + if let Ok(v) = serde_json::from_str::(&hit) { + if !v.ranked.is_empty() { + return v; + } + } + } + } + + let mut ranked: Vec<(String, String)> = Vec::new(); + let deadline = Duration::from_secs(MB_TIMEOUT_SECS); + + // Did the MusicBrainz leg actually *answer*? Not "did it find + // anything" — see the caching rule at the bottom of this function. + let mut mb_answered = artist_mbid.is_empty(); + + if let Some(mb) = self.mb.as_ref() { + if !artist_mbid.is_empty() { + if let Some(rels) = mb.artist_url_relations(artist_mbid, deadline).await { + mb_answered = true; + // `youtube music` first, `youtube` second — the order is a + // measurement, not a preference. See `ChannelKey`. + for (want, key) in [ + ("youtube music", ChannelKey::MbYoutubeMusic), + ("youtube", ChannelKey::MbYoutube), + ] { + for (ty, url) in &rels { + if ty.eq_ignore_ascii_case(want) { + if let Some(id) = channel_id_from_url(url) { + ranked.push((id, key.as_str().to_string())); + } + } + } + } + } + } + } + + // The third key. Tried even when MusicBrainz answered, because it costs + // one search and covers the 4-of-12 albums MusicBrainz has no relation + // for at all. Ordered last, so MusicBrainz still wins where it has an + // answer — and where both answer they agree (Daft Punk resolves to + // `UCRr1xG_2WIDs18a6cIiCxeA` either way). + // + // Tried under every name MusicBrainz knows, not just the credited one: + // a non-Latin credit finds no exact-name channel even when the artist + // plainly has one (§7.2.1, `植松伸夫` vs `Nobuo Uematsu`). + let mut names = vec![artist_name.to_string()]; + if let Some(mb) = self.mb.as_ref() { + if !artist_mbid.is_empty() { + names.extend(mb.artist_names(artist_mbid, deadline).await); + } + } + for name in names { + if let Some(id) = self.piped_artist_channel(&name).await { + if !ranked.iter().any(|(c, _)| *c == id) { + ranked.push((id, ChannelKey::PipedMusicArtists.as_str().to_string())); + } + break; + } + } + + let keys = ChannelKeys { ranked }; + + // ⚠ **Cache only a COMPLETE answer** — and "complete" means the + // MusicBrainz leg answered, not that the list came back non-empty. + // + // This is the §7.5 trap in the form that actually bit, caught in a live + // smoke test rather than by reasoning. MusicBrainz answered `503` + // (`musicbrainz throttled; pausing budget`), `artist_url_relations` + // returned `None` like every other failure, the Piped key still + // succeeded — and a **non-empty but partial** list got written to a + // permanent cache. Daft Punk, which has both a `youtube` and a + // `youtube music` relation, was pinned to `piped:music-artists` + // forever: still resolving, still correct, but recorded under the + // weakest key and unable to ever improve. + // + // Guarding on `!ranked.is_empty()` alone does not catch this, because + // the degraded answer is not empty. The condition has to be about + // whether the *authoritative* source spoke. + if mb_answered && !keys.ranked.is_empty() { + if let (Some(c), Ok(json)) = (self.cache.as_ref(), serde_json::to_string(&keys)) { + let _ = c.put_yt_channel(cache_key, &json); + } + } else if !mb_answered { + debug!( + target: "meta-music", + artist = %artist_name, + "youtube: musicbrainz did not answer; using channel keys but NOT caching them" + ); + } + keys + } + + /// Piped's `music_artists` index — the coverage-gap filler. + /// + /// Results are subscriber-ordered, so the first exact-name match is right + /// in practice even where two channels share a name (two "Burial" channels + /// exist: 37 k subscribers and 39). + async fn piped_artist_channel(&self, artist_name: &str) -> Option { + let name = artist_name.trim(); + if name.is_empty() { + return None; + } + let v = self + .piped_get(&format!( + "search?q={}&filter=music_artists", + urlencoding_encode(name) + )) + .await?; + let items = v.get("items")?.as_array()?; + for it in items { + let uploader = it + .get("name") + .and_then(|n| n.as_str()) + .unwrap_or_default(); + if normalise_title(uploader) == normalise_title(name) { + if let Some(id) = it.get("url").and_then(|u| u.as_str()).and_then(channel_id_from_url) { + return Some(id); + } + } + } + None + } + + /// The artist's **music videos** — the video tier of the same delegation. + /// + /// # Why the correctness rule is different here, and why that is sound + /// + /// A track is checked against a MusicBrainz *recording*: title, and a + /// `length` to separate the album cut from the radio edit. A music video + /// has no such anchor — MusicBrainz does not model one, so there is no + /// canonical list to be wrong against. + /// + /// So the whole claim this makes is narrower, and the channel match carries + /// it completely: **"this video was uploaded by the artist's own official + /// channel."** That is verifiable exactly, it is the same channel key the + /// track path already resolved, and nothing weaker is asserted — no + /// "this is the video *of* that recording", which is the claim that would + /// need an anchor we do not have. + async fn music_videos( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + let Some(artist_mbid) = query + .filters + .get("mbArtistId") + .and_then(|v| v.first()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + else { + // Id-anchored, exactly as the track path is: there is no free-text + // browse to fall back to, and answering one would fill an artist's + // row with somebody else's videos. + return Ok(Vec::new()); + }; + let Some(mb) = self.mb.as_ref() else { + return Ok(Vec::new()); + }; + let deadline = Duration::from_secs(MB_TIMEOUT_SECS); + let Some(artist) = mb.artist(&artist_mbid, deadline).await else { + return Ok(Vec::new()); + }; + + let channels = self.channel_keys(&artist_mbid, &artist.name).await; + if channels.ranked.is_empty() { + debug!( + target: "meta-music", + artist = %artist.name, + "youtube: no channel key; music videos would be unverifiable" + ); + return Ok(Vec::new()); + } + + let Some(v) = self + .piped_get(&format!( + "search?q={}&filter=music_videos", + urlencoding_encode(&artist.name) + )) + .await + else { + return Ok(Vec::new()); + }; + + let mut out = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for c in parse_candidates(&v) { + if out.len() >= max_results { + break; + } + let Some((_, key)) = channels + .ranked + .iter() + .find(|(id, _)| *id == c.uploader_channel) + else { + continue; // not this artist's channel — the entire filter + }; + if !seen.insert(c.video_id.clone()) { + continue; + } + if !self.is_embeddable(&c.video_id).await { + continue; + } + if let Some(rec) = + self.music_video_record(&artist_mbid, &artist.name, &c, key, &query.filters) + { + out.push(rec); + } + } + Ok(out) + } + + /// Every EXACT candidate for one track, best first. + /// + /// Rank is duration delta, then channel-key strength — a MusicBrainz + /// `youtube music` relation is a stronger claim than a name match against + /// Piped's own artist index, so it breaks ties. + async fn exact_candidates( + &self, + artist: &str, + track: &musicbrainz_feeder::client::Track, + want_secs: Option, + channels: &ChannelKeys, + ) -> Vec { + let mut cands = self.search_track(artist, &track.title).await; + + // §7.5 — MusicBrainz's filed title is often not the title anyone says + // (`Lift Yr. Skinny Fists…`, `MAP OF THE SOUL : 7`), so a single + // exact-phrase query misses. Retry once on a normalised title before + // concluding the corpus has nothing. + if !cands.iter().any(|c| { + grade(c, &track.title, want_secs, &channels.ranked).0 == Verdict::Exact + }) { + let loose = normalise_title(&track.title); + if !loose.is_empty() && loose != track.title.to_lowercase() { + cands.extend(self.search_track(artist, &loose).await); + } + } + + let mut out: Vec = Vec::new(); + for c in cands { + let (verdict, key) = grade(&c, &track.title, want_secs, &channels.ranked); + if verdict != Verdict::Exact { + continue; + } + let delta = want_secs.map(|w| (c.duration_secs - w).abs()).unwrap_or(i64::MAX); + if out.iter().any(|r| r.video_id == c.video_id) { + continue; // the two queries can return the same upload + } + out.push(TrackRef { + video_id: c.video_id, + title: c.title, + match_key: key.unwrap_or_default(), + delta, + }); + } + out.sort_by_key(|r| (r.delta, key_priority(&r.match_key))); + out + } + + /// Is this video embeddable **here**, right now? + /// + /// YouTube's oEmbed endpoint needs no key, no quota and no account: + /// + /// ```text + /// 200 -> embeddable 401 -> embedding denied or private + /// 404 -> gone + /// ``` + /// + /// This is what moves a dead reference from a play-time error 101/150 — + /// the worst place to discover it — to resolve time, where it simply is not + /// emitted. The Data API could report `regionRestriction` too, but it + /// cannot be bought at any price (10k units/day, `search.list` = 100 units), + /// so oEmbed carries the ubiquitous check and the Data API carries nothing. + /// + /// **Positives are cached, negatives are not** — same rule as the channel + /// keys. A `401`/`404` is a real verdict, but a transport failure looks + /// identical from here, and freezing "this video is dead" into a permanent + /// cache on a network hiccup is the §7.5 trap wearing another hat. + async fn is_embeddable(&self, video_id: &str) -> bool { + let key = format!("oembed:{video_id}"); + if let Some(c) = self.cache.as_ref() { + if matches!(c.get_yt_search(&key), Ok(Some(_))) { + return true; + } + } + let url = format!( + "https://www.youtube.com/oembed?url={}&format=json", + urlencoding_encode(&format!("https://www.youtube.com/watch?v={video_id}")) + ); + let ok = match self.http.get(&url).send().await { + Ok(r) => r.status().is_success(), + Err(e) => { + debug!(target: "meta-music", %video_id, error = %e, "oembed probe failed; treating as embeddable"); + // A failed probe is not a negative verdict. Emitting on doubt is + // the right side to err on: the client already degrades a dead + // reference gracefully (it notes and skips), whereas dropping a + // good one is silent. + return true; + } + }; + if ok { + if let Some(c) = self.cache.as_ref() { + let _ = c.put_yt_search(&key, "1"); + } + } + ok + } + + /// Search `music_songs` for one track. + /// + /// `filter=music_songs` matters: it queries the YouTube *Music* corpus, so + /// results are official audio rather than the video-first ranking of a + /// plain search. + async fn search_track(&self, artist: &str, title: &str) -> Vec { + let q = format!("{artist} {title}"); + if let Some(c) = self.cache.as_ref() { + if let Ok(Some(hit)) = c.get_yt_search(&meta_feeder_sdk::cache::search_key("yt", &q)) { + if let Ok(v) = serde_json::from_str::(&hit) { + return parse_candidates(&v); + } + } + } + let Some(v) = self + .piped_get(&format!( + "search?q={}&filter=music_songs", + urlencoding_encode(&q) + )) + .await + else { + // A failed search is NOT an empty one. Returning `[]` here would be + // correct-looking and wrong: it would let a Piped outage present as + // "this album has no YouTube versions". Nothing is cached, so the + // next attempt retries. + return Vec::new(); + }; + if let (Some(c), Ok(json)) = (self.cache.as_ref(), serde_json::to_string(&v)) { + let _ = c.put_yt_search(&meta_feeder_sdk::cache::search_key("yt", &q), &json); + } + parse_candidates(&v) + } +} + +/// One recording's ranked, embeddable references, before global de-confliction. +struct TrackPick { + recording_mbid: String, + track: musicbrainz_feeder::client::Track, + /// Best first: smaller duration delta wins, then the stronger channel key. + refs: Vec, +} + +/// One EXACT reference, carrying what ranking and the record both need. +#[derive(Clone, Debug)] +struct TrackRef { + video_id: String, + title: String, + /// Which [`ChannelKey`] produced the match, as written to `ytMatchKey`. + match_key: String, + /// |candidate runtime − MusicBrainz length|, in seconds. The primary rank. + delta: i64, +} + +/// **A video may belong to only one recording.** +/// +/// In the study Burial's *Archangel* and *Near Dark* both resolved to +/// `VVa_aCSA9Jc` at EXACT — same channel, both durations inside tolerance. +/// They are different recordings, so at least one is wrong: a collision is +/// **proof of a bad match**, not a duplicate to tidy away. The tighter duration +/// delta keeps the claim and the weaker one drops the reference entirely rather +/// than emitting both. +/// +/// A pure function over the gathered picks, deliberately: the previous form +/// mutated the output vector mid-loop and hand-patched stored indices, which is +/// exactly the shape that breaks silently once a track can contribute more than +/// one reference. +fn assign_references(picks: &mut [TrackPick]) { + // video_id -> (pick index, winning delta) + let mut claimed: HashMap = HashMap::new(); + for (i, pick) in picks.iter().enumerate() { + for r in &pick.refs { + match claimed.get(&r.video_id) { + Some(&(_, best)) if best <= r.delta => {} + _ => { + claimed.insert(r.video_id.clone(), (i, r.delta)); + } + } + } + } + for (i, pick) in picks.iter_mut().enumerate() { + pick.refs + .retain(|r| claimed.get(&r.video_id).is_some_and(|&(owner, _)| owner == i)); + } +} + +/// Grade one candidate against a track. See [`Verdict`]. +fn grade( + cand: &Candidate, + track_title: &str, + mb_length_secs: Option, + channels: &[(String, String)], +) -> (Verdict, Option) { + let channel_hit = channels + .iter() + .find(|(id, _)| *id == cand.uploader_channel) + .map(|(_, key)| key.clone()); + + let Some(key) = channel_hit else { + return (Verdict::NoMatch, None); + }; + + // Duration is what separates the right recording from the right artist. + // A track MusicBrainz never timed cannot reach EXACT: with no length there + // is nothing to compare, and "the channel agreed" alone is precisely the + // classical trap. + let Some(want) = mb_length_secs else { + return (Verdict::ChannelWrongCut, Some(key)); + }; + if (cand.duration_secs - want).abs() > YT_DURATION_TOLERANCE_SECS { + return (Verdict::ChannelWrongCut, Some(key)); + } + + // Title agreement, on top of channel+duration. Duration alone is a coin + // flip wearing a plausible runtime — two real misses from the study, both + // inside one second: Burial "Archangel" -> "Near Dark" (+1 s), Adele + // "Don't You Remember" -> "Set Fire to the Rain" (+0 s). + if !titles_agree(&cand.title, track_title) { + return (Verdict::ChannelWrongCut, Some(key)); + } + + (Verdict::Exact, Some(key)) +} + +impl YouTubePlugin { + /// One `contentKind:musicVideo` record. + /// + /// `fileType:video`, not `audio` — this is the video tier, and meta-listen's + /// artist row filters on exactly that pair. + fn music_video_record( + &self, + artist_mbid: &str, + artist: &str, + c: &Candidate, + match_key: &str, + query_filters: &BTreeMap>, + ) -> Option { + let yt_cid = meta_feeder_sdk::hash::compute_yt_video_cid("video", &c.video_id)?; + + let mut fields: BTreeMap = BTreeMap::new(); + fields.insert("fileType".to_string(), "video".to_string()); + fields.insert("contentKind".to_string(), "musicVideo".to_string()); + fields.insert("title".to_string(), c.title.clone()); + fields.insert("artist".to_string(), artist.to_string()); + fields.insert(format!("artists/{}", artist.trim()), "true".to_string()); + fields.insert("mbArtistId".to_string(), artist_mbid.to_string()); + if c.duration_secs > 0 { + fields.insert("fileinfo/duration".to_string(), c.duration_secs.to_string()); + } + fields.insert(format!("cids/{yt_cid}"), "true".to_string()); + fields.insert("ytMatchKey".to_string(), match_key.to_string()); + fields.insert("youtubeid".to_string(), c.video_id.clone()); + fields.insert("source/gateway:youtube".to_string(), "true".to_string()); + + // `gate::keep` requires cover + artist for a musicVideo, and a missing + // cover drops the record silently — the same trap as the track path. + if let Some(locator) = meta_feeder_sdk::hash::artwork_locator(&format!( + "{YT_THUMB_BASE}/{}/hqdefault.jpg", + c.video_id + )) { + fields.insert("cover".to_string(), locator); + } + + echo_filters(&mut fields, query_filters); + Some(DiscoveryRecord { + upstream_id: "youtube".to_string(), + record_id: track_record_id(artist_mbid, &c.video_id), + fields, + }) + } +} + +/// Rank order for [`ChannelKey`] as a tie-break: authoritative first. +fn key_priority(match_key: &str) -> u8 { + match match_key { + "mb:youtube-music" => 0, + "mb:youtube" => 1, + _ => 2, + } +} + +/// Do two titles name the same recording? +/// +/// Two rules, and the asymmetry between the costs is what shapes them: a false +/// *refusal* costs one row of coverage, a false *accept* puts the wrong song +/// under the right name. So this stays deliberately tight. +/// +/// 1. **Containment, either direction** — because YouTube Music decorates a +/// title with the feature credits MusicBrainz keeps in the artist credit +/// ("Get Lucky" vs "Get Lucky (feat. Pharrell Williams and Nile Rodgers)"). +/// 2. **High token overlap on longer titles** — because MusicBrainz's filed +/// title is often not the title anyone says, and the difference is a single +/// word: `Lift Yr. Skinny Fists…` against `Lift Your Skinny Fists…` shares +/// nine tokens of ten but fails containment outright. +/// +/// Rule 2 is gated at [`TITLE_MIN_TOKENS`] tokens precisely so it cannot loosen +/// short titles, where a single shared word is most of the title and overlap +/// stops discriminating. +fn titles_agree(a: &str, b: &str) -> bool { + let (na, nb) = (normalise_title(a), normalise_title(b)); + if na.is_empty() || nb.is_empty() { + return false; + } + if na.contains(&nb) || nb.contains(&na) { + return true; + } + let ta: std::collections::BTreeSet<&str> = na.split(' ').collect(); + let tb: std::collections::BTreeSet<&str> = nb.split(' ').collect(); + let smaller = ta.len().min(tb.len()); + if smaller < TITLE_MIN_TOKENS { + return false; + } + let shared = ta.intersection(&tb).count(); + shared as f32 / smaller as f32 >= TITLE_MIN_OVERLAP +} + +/// Token count below which [`titles_agree`] rule 2 does not apply. +const TITLE_MIN_TOKENS: usize = 4; +/// Fraction of the shorter title's tokens that must be shared under rule 2. +const TITLE_MIN_OVERLAP: f32 = 0.8; + +/// Lowercase, drop parentheticals, strip non-alphanumerics, collapse spaces. +/// +/// MusicBrainz's title is often not the title anyone says — `Lift Yr. Skinny +/// Fists Like Antennas to Heaven!` and `MAP OF THE SOUL : 7` are the filed +/// forms — so an exact-phrase comparison misses both. +fn normalise_title(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut depth = 0usize; + for ch in s.chars() { + match ch { + '(' | '[' => depth += 1, + ')' | ']' => depth = depth.saturating_sub(1), + _ if depth > 0 => {} + c if c.is_alphanumeric() => out.extend(c.to_lowercase()), + _ => out.push(' '), + } + } + out.split_whitespace().collect::>().join(" ") +} + +/// Extract a bare channel id from any of the URL shapes MusicBrainz and Piped +/// use (`https://www.youtube.com/channel/UC…`, `/channel/UC…`). +fn channel_id_from_url(url: &str) -> Option { + let (_, tail) = url.split_once("/channel/")?; + let id = tail.split(['/', '?', '#']).next()?.trim(); + (!id.is_empty()).then(|| id.to_string()) +} + +/// Extract a bare video id from Piped's `/watch?v=` url field. +fn video_id_from_url(url: &str) -> Option { + let (_, tail) = url.split_once("v=")?; + let id = tail.split(['&', '#']).next()?.trim(); + (!id.is_empty()).then(|| id.to_string()) +} + +fn parse_candidates(v: &Value) -> Vec { + let Some(items) = v.get("items").and_then(|i| i.as_array()) else { + return Vec::new(); + }; + items + .iter() + .take(YT_CANDIDATES_PER_TRACK) + .filter_map(|it| { + Some(Candidate { + video_id: video_id_from_url(it.get("url")?.as_str()?)?, + title: it.get("title")?.as_str()?.to_string(), + uploader_channel: it + .get("uploaderUrl") + .and_then(|u| u.as_str()) + .and_then(channel_id_from_url) + .unwrap_or_default(), + duration_secs: it.get("duration").and_then(|d| d.as_i64()).unwrap_or(-1), + }) + }) + .collect() +} + +/// Minimal percent-encoder for a query-string value. +fn urlencoding_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len() * 3); + for b in s.as_bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*b as char) + } + b' ' => out.push('+'), + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +/// `~` — the record id this plugin mints. +/// +/// # Why the video id is *in* the id +/// +/// It makes [`YouTubePlugin::compute_outcomes`] a **pure function with zero +/// I/O**, exactly as `compute_card_cid` is for the card feeder. The alternative +/// — keying only on the recording and re-resolving at outcome time — would +/// re-run a MusicBrainz tracklist, a channel lookup and a Piped search, and +/// could legitimately pick a *different* video than the one the consumer is +/// looking at. Minting a reference other than the one on screen is worse than +/// slow. +/// +/// It also gives the multi-reference design its natural shape: several ranked +/// references for one recording are several records, each stably addressed, +/// each converging across peers via the `cids/` reverse index — rather than one +/// record whose meaning silently changes when the resolver's pick moves. +pub fn track_record_id(recording_mbid: &str, video_id: &str) -> String { + format!("{recording_mbid}~{video_id}") +} + +/// Inverse of [`track_record_id`] — `(recording_mbid, video_id)`. +pub fn split_track_record_id(record_id: &str) -> Option<(&str, &str)> { + let (rec, vid) = record_id.split_once('~')?; + (!rec.is_empty() && !vid.is_empty()).then_some((rec, vid)) +} + +#[async_trait] +impl FeederPlugin for YouTubePlugin { + fn upstream_id(&self) -> &'static str { + "youtube" + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["audio", "video"] + } + + /// `track` and `musicVideo` — deliberately **not** `pack`. + /// + /// A pack is a container whose members are separately addressable, and + /// there is no such thing as a delegated *album*: each reference is one + /// video. Claiming `pack` would advertise a capability that can only + /// answer with a synthetic container holding nothing fetchable. + /// + /// `musicVideo` is the *video* tier of the same delegation, and it is what + /// finally fills meta-listen's artist-page "Music videos" row — empty until + /// now because nothing in the platform produced the content kind. YouTube + /// hosts the Art Track and the music video as **separate video ids**, which + /// is exactly what makes the YouTube-Music-style song/video toggle + /// implementable: swap the reference, keep the position. + fn served_content_kinds(&self) -> &'static [&'static str] { + &["track", "musicVideo"] + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + let cache = MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin: "youtube", + source: anyhow::anyhow!("open music cache: {e}"), + })?; + // Its own MusicBrainz budget rather than a shared one: MusicBrainz's + // 1 req/s ceiling is enforced per source IP, and a budget shared with + // the card plugin would let a board refresh and an album open burst + // past it together. + let budget = RateBudget::new(MB_RATE_PER_SEC, MB_BURST); + let mb = match &self.mb_base { + Some(base) => MusicBrainzClient::with_base(base.clone(), budget, cache.clone()), + None => MusicBrainzClient::new(budget, cache.clone()), + }; + self.mb = Some(Arc::new(mb)); + self.cache = Some(cache); + Ok(()) + } + + fn health(&self) -> PluginHealth { + match self.mb { + Some(_) => PluginHealth::Ok, + None => PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + }, + } + } + + /// Resolve one release-group's tracks to YouTube references. + /// + /// # This plugin is the mirror image of `internetarchive` + /// + /// The archive holds no MusicBrainz ids, so it must return **nothing** for + /// an id-anchored query. This plugin can *only* work from one: you cannot + /// search YouTube for a UUID, so the chain is necessarily + /// `mbReleaseGroupId` → tracklist (titles + `length` + `mbRecordingId`) → + /// artist channel → per-track search → filter. A free-text query therefore + /// returns nothing here — not because it would be expensive, but because + /// the resolver has no anchor to verify a match against, and an unverified + /// match is the one output this tier must never produce. + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + // Two anchors, two answers. The music-video row asks + // `mbArtistId: fileType:video contentKind:musicVideo`, which is a + // different question from an album page's + // `mbReleaseGroupId: fileType:audio` — different entity, different + // corpus, different correctness criterion. Dispatch on which anchor is + // present rather than trying to serve both from one path. + if wants_content_kind(query, "musicVideo") && !wants_content_kind(query, "track") { + return self.music_videos(query, max_results).await; + } + if !wants_content_kind(query, "track") { + return Ok(Vec::new()); + } + + let Some(rg_mbid) = query + .filters + .get("mbReleaseGroupId") + .and_then(|v| v.first()) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + else { + debug!( + target: "meta-music", + "youtube: query carries no mbReleaseGroupId; nothing to anchor on" + ); + return Ok(Vec::new()); + }; + + let Some(mb) = self.mb.as_ref() else { + return Ok(Vec::new()); + }; + let deadline = Duration::from_secs(MB_TIMEOUT_SECS); + + let Some(release) = mb.tracklist(&rg_mbid, deadline).await else { + // Not "this album has no tracks" — the lookup did not answer. Never + // conflate the two (study §7.5); emit nothing and let the next + // attempt retry. + debug!(target: "meta-music", rg = %rg_mbid, "youtube: no tracklist"); + return Ok(Vec::new()); + }; + + // Whose channel to look for. The release-group's credited artist is the + // right *starting* point, but see `Verdict::ChannelWrongCut`: for + // classical this is the composer, and the duration filter is what stops + // that from resolving to someone else's performance. + let rg = mb.release_group(&rg_mbid, deadline).await; + let (artist_mbid, artist_name) = rg + .as_ref() + .and_then(|g| primary_artist(&g.artist_credit)) + .map(|a| (a.id.clone(), a.name.clone())) + .unwrap_or_default(); + + let channels = self.channel_keys(&artist_mbid, &artist_name).await; + if channels.ranked.is_empty() { + debug!( + target: "meta-music", + artist = %artist_name, + "youtube: no channel key for this artist; every track would be unverifiable" + ); + return Ok(Vec::new()); + } + + let album_title = release.title.clone(); + + // ── 1. Gather ranked EXACT candidates per track ────────────────────── + // + // Ranked, not reduced to a single best: §3's rot mitigation is to carry + // *several* references per recording so a client can fall through when + // one is taken down, geo-blocked or has embedding disabled. A single + // reference means a takedown kills the track outright. + let mut picks: Vec = Vec::new(); + for medium in &release.media { + for track in &medium.tracks { + let recording_mbid = track + .recording + .as_ref() + .map(|r| r.id.clone()) + .unwrap_or_default(); + if recording_mbid.is_empty() { + // Without it the consumer matches versions to track rows by + // fuzzy title and lands on the wrong row for anything with a + // reprise, a remix or a live variant. + continue; + } + let want_secs = track + .length + .or_else(|| track.recording.as_ref().and_then(|r| r.length)) + .map(|ms| (ms / 1000) as i64); + + let mut refs = self.exact_candidates(&artist_name, track, want_secs, &channels).await; + // Probe only what could plausibly be emitted — one spare, so a + // single dead reference does not cost the track a slot. + refs.truncate(MAX_REFS_PER_RECORDING + 1); + let mut embeddable = Vec::new(); + for r in refs { + if self.is_embeddable(&r.video_id).await { + embeddable.push(r); + } + } + if embeddable.is_empty() { + continue; + } + picks.push(TrackPick { + recording_mbid, + track: track.clone(), + refs: embeddable, + }); + } + } + + // ── 2. One video may belong to only one recording ──────────────────── + assign_references(&mut picks); + + // ── 3. Emit ───────────────────────────────────────────────────────── + let mut out: Vec = Vec::new(); + for pick in &picks { + for (rank, r) in pick.refs.iter().take(MAX_REFS_PER_RECORDING).enumerate() { + if out.len() >= max_results { + break; + } + if let Some(rec) = self.track_record( + &rg_mbid, + &pick.recording_mbid, + &album_title, + &artist_name, + &pick.track, + r, + rank as u32, + &query.filters, + ) { + out.push(rec); + } + } + } + + out.truncate(max_results); + Ok(out) + } + + /// Metadata-only: a reference, never bytes. + /// + /// The `(None, Some)` branch — `bytes: None`, so the core auto-stores the + /// record and stops. There is no blockstore seed (gated on `Sha2_256`), no + /// WebDAV blob, and no `file_extension` (which MUST be `None` when `bytes` + /// is). + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let (_recording_mbid, video_id) = split_track_record_id(record_id).ok_or_else(|| { + GatewayError::Permanent(format!("malformed youtube record_id '{record_id}'")) + })?; + let hash = meta_feeder_sdk::hash::compute_yt_video_cid("video", video_id).ok_or_else( + || GatewayError::Permanent(format!("video id too long to encode: '{video_id}'")), + )?; + + // `record: None` — unlike the card feeder there is nothing to + // re-resolve. `handle_query` already wrote the full record, and the + // only thing this cid could be re-derived from is the video id already + // in the record_id. + Ok(vec![HashOutcome { + hash: Hash(hash), + hash_kind: HashKind::YtVideo, + bytes: None, + record: None, + file_extension: None, + }]) + } + + /// Never. This tier has no byte path by construction. + async fn handle_fetch(&self, _record_id: &str) -> Result, GatewayError> { + Ok(None) + } +} + +impl YouTubePlugin { + /// Build one delegated-playback track record. + /// + /// Returns `None` if the `yt-video` cid cannot be minted — a record with no + /// `cids/*` member is unplayable and would only add a dead row. + #[allow(clippy::too_many_arguments)] + fn track_record( + &self, + rg_mbid: &str, + recording_mbid: &str, + album: &str, + artist: &str, + track: &musicbrainz_feeder::client::Track, + r: &TrackRef, + rank: u32, + query_filters: &BTreeMap>, + ) -> Option { + let yt_cid = meta_feeder_sdk::hash::compute_yt_video_cid("video", &r.video_id)?; + + let mut fields: BTreeMap = BTreeMap::new(); + fields.insert("fileType".to_string(), "audio".to_string()); + fields.insert("contentKind".to_string(), "track".to_string()); + fields.insert("title".to_string(), track.title.clone()); + fields.insert("album".to_string(), album.to_string()); + if !artist.trim().is_empty() { + fields.insert("albumArtist".to_string(), artist.to_string()); + fields.insert("artist".to_string(), artist.to_string()); + fields.insert(format!("artists/{}", artist.trim()), "true".to_string()); + } + if track.position > 0 { + fields.insert("trackNumber".to_string(), track.position.to_string()); + } + // Matched on by the consumer to place a version against a track row. + fields.insert("mbRecordingId".to_string(), recording_mbid.to_string()); + fields.insert("mbReleaseGroupId".to_string(), rg_mbid.to_string()); + + // The reference itself. + fields.insert(format!("cids/{yt_cid}"), "true".to_string()); + // Which key produced the channel match. See `ChannelKey`. + fields.insert("ytMatchKey".to_string(), r.match_key.clone()); + // Which fallback position this reference occupies for the recording, + // 0 = best. Several references per recording is the mitigation for + // reference rot (§3): the id is immutable but its *availability* is + // not, so the client needs a defined order to fall through. + fields.insert("ytRank".to_string(), rank.to_string()); + + // The canonical `id` field every feeder record must carry. + fields.insert("youtubeid".to_string(), r.video_id.clone()); + fields.insert("source/gateway:youtube".to_string(), "true".to_string()); + + // The cover — load-bearing, and silently fatal if omitted. + // + // The consumer's gate requires artwork (`gate::has_cover`), so a record + // without one is dropped before it ever reaches a version list, with no + // error anywhere. Emitted as a `url` LOCATOR (`0x1006`), which is the + // current convention: `cover_url` was REMOVED on 2026-08-21 because a + // `*_url` field only looks transient — the gateway rewrites it on + // success, so a 404 leaves the raw URL persisted forever. + // + // Note this is `cover`, not `poster`: music artwork has its own + // vocabulary since the two images come from different places + // (METADATA_KEYS.md §6). + if let Some(locator) = meta_feeder_sdk::hash::artwork_locator(&format!( + "{YT_THUMB_BASE}/{}/hqdefault.jpg", + r.video_id + )) { + fields.insert("cover".to_string(), locator); + } + + echo_filters(&mut fields, query_filters); + Some(DiscoveryRecord { + upstream_id: "youtube".to_string(), + record_id: track_record_id(recording_mbid, &r.video_id), + fields, + }) + } +} + +/// Local copies of the two tiny query helpers `internetarchive` also uses. +/// Duplicated rather than shared because they are three lines each and the +/// alternative is a `pub` surface on a sibling plugin module. +fn wants_content_kind(query: &GatewayQuery, kind: &str) -> bool { + match query.filters.get("contentKind") { + None => true, + Some(values) if values.is_empty() => true, + Some(values) => values.iter().any(|v| v.trim().eq_ignore_ascii_case(kind)), + } +} + +fn echo_filters(fields: &mut BTreeMap, query_filters: &BTreeMap>) { + for (key, allowed) in query_filters { + if key == "languages" || key == "genres" || fields.contains_key(key) || allowed.is_empty() { + continue; + } + fields.insert(key.clone(), allowed.join(",")); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chan(id: &str) -> Vec<(String, String)> { + vec![(id.to_string(), ChannelKey::MbYoutubeMusic.as_str().to_string())] + } + + fn cand(video: &str, title: &str, channel: &str, secs: i64) -> Candidate { + Candidate { + video_id: video.to_string(), + title: title.to_string(), + uploader_channel: channel.to_string(), + duration_secs: secs, + } + } + + #[test] + fn exact_needs_channel_and_duration_and_title() { + // The real Daft Punk case from the study: MB "Get Lucky" at 370 s, the + // YouTube Music upload decorated with the feature credits. + let c = cand( + "4D7u5KF7SP8", + "Get Lucky (feat. Pharrell Williams and Nile Rodgers)", + "UCRr1xG_2WIDs18a6cIiCxeA", + 370, + ); + let (v, key) = grade(&c, "Get Lucky", Some(370), &chan("UCRr1xG_2WIDs18a6cIiCxeA")); + assert_eq!(v, Verdict::Exact); + assert_eq!(key.as_deref(), Some("mb:youtube-music")); + } + + #[test] + fn a_different_channel_is_never_a_match() { + let c = cand("x", "Get Lucky", "UCsomeoneelse", 370); + let (v, _) = grade(&c, "Get Lucky", Some(370), &chan("UCRr1xG_2WIDs18a6cIiCxeA")); + assert_eq!(v, Verdict::NoMatch); + } + + #[test] + fn channel_match_with_far_off_duration_is_a_refusal() { + // THE classical trap, with the study's measured deltas. MusicBrainz + // credits Bach for a Glenn Gould recording, so the channel agrees + // confidently and returns someone else's performance (+32 s, +209 s). + for delta in [32, 209] { + let c = cand("v", "The Goldberg Variations", "UCbach", 1000 + delta); + let (v, key) = grade(&c, "The Goldberg Variations", Some(1000), &chan("UCbach")); + assert_eq!( + v, + Verdict::ChannelWrongCut, + "a +{delta}s channel match must refuse, never widen into a near-miss" + ); + // The key is still reported so the refusal is attributable. + assert!(key.is_some()); + } + } + + #[test] + fn a_track_musicbrainz_never_timed_cannot_reach_exact() { + // No length => nothing to compare => "the channel agreed" alone, which + // is exactly the shape the classical trap produces. + let c = cand("v", "Some Track", "UCartist", 200); + let (v, _) = grade(&c, "Some Track", None, &chan("UCartist")); + assert_eq!(v, Verdict::ChannelWrongCut); + } + + #[test] + fn duration_alone_is_not_enough_the_title_must_agree() { + // Both real misses from the study's first run, both inside one second. + let burial = cand("v1", "Near Dark", "UCburial", 241); + let (v, _) = grade(&burial, "Archangel", Some(240), &chan("UCburial")); + assert_eq!(v, Verdict::ChannelWrongCut); + + let adele = cand("v2", "Set Fire to the Rain", "UCadele", 242); + let (v, _) = grade(&adele, "Don't You Remember", Some(242), &chan("UCadele")); + assert_eq!(v, Verdict::ChannelWrongCut); + } + + #[test] + fn title_normalisation_survives_musicbrainz_filing_conventions() { + // MusicBrainz's title is often not the title anyone says. + assert!(titles_agree( + "Lift Your Skinny Fists Like Antennas to Heaven", + "Lift Yr. Skinny Fists Like Antennas to Heaven!" + )); + assert!(titles_agree("MAP OF THE SOUL : 7", "Map of the Soul: 7")); + // ...but it is not so loose that two different songs agree. + assert!(!titles_agree("Archangel", "Near Dark")); + assert!(!titles_agree("Don't You Remember", "Set Fire to the Rain")); + // The short-title guard: overlap must NOT rescue a 3-token near-miss, + // where one shared word is most of the title. + assert!(!titles_agree("Love Me Tender", "Love Me Do")); + } + + #[test] + fn channel_and_video_ids_are_extracted_from_every_url_shape() { + assert_eq!( + channel_id_from_url("https://www.youtube.com/channel/UC_kRDKYrUlrbtrSiyu5Tflg").as_deref(), + Some("UC_kRDKYrUlrbtrSiyu5Tflg") + ); + assert_eq!( + channel_id_from_url("https://music.youtube.com/channel/UCRr1xG_2WIDs18a6cIiCxeA") + .as_deref(), + Some("UCRr1xG_2WIDs18a6cIiCxeA") + ); + assert_eq!(channel_id_from_url("/channel/UCabc?x=1").as_deref(), Some("UCabc")); + assert!(channel_id_from_url("https://youtube.com/user/foo").is_none()); + + assert_eq!(video_id_from_url("/watch?v=4D7u5KF7SP8").as_deref(), Some("4D7u5KF7SP8")); + assert_eq!( + video_id_from_url("/watch?v=4D7u5KF7SP8&list=OLAK").as_deref(), + Some("4D7u5KF7SP8") + ); + } + + #[test] + fn record_id_round_trips_and_mints_the_cid_with_no_io() { + let id = track_record_id("345e4a72-46b4-48ba-8541-27c6b23c3b8c", "4D7u5KF7SP8"); + let (rec, vid) = split_track_record_id(&id).expect("round-trips"); + assert_eq!(rec, "345e4a72-46b4-48ba-8541-27c6b23c3b8c"); + assert_eq!(vid, "4D7u5KF7SP8"); + // The same cid the golden vector pins. + assert_eq!( + meta_feeder_sdk::hash::compute_yt_video_cid("video", vid).unwrap(), + "bagecaaarav3gszdfn42ein3vgvfumn2tka4a" + ); + assert!(split_track_record_id("no-separator").is_none()); + assert!(split_track_record_id("rec~").is_none()); + } + + #[tokio::test] + async fn compute_outcomes_is_metadata_only() { + let p = YouTubePlugin::default(); + let id = track_record_id("rec-mbid", "4D7u5KF7SP8"); + let out = p.compute_outcomes(&id).await.expect("outcome"); + assert_eq!(out.len(), 1); + assert_eq!(out[0].hash_kind, HashKind::YtVideo); + // The three properties that make this tier what it is. + assert!(out[0].bytes.is_none(), "delegated playback transports no bytes"); + assert!( + out[0].file_extension.is_none(), + "file_extension MUST be None when bytes is None" + ); + assert_eq!(out[0].hash.0, "bagecaaarav3gszdfn42ein3vgvfumn2tka4a"); + } + + #[tokio::test] + async fn handle_fetch_never_returns_a_stream() { + let p = YouTubePlugin::default(); + assert!(p + .handle_fetch(&track_record_id("rec", "vid")) + .await + .expect("ok") + .is_none()); + } + + #[tokio::test] + async fn a_query_with_no_release_group_anchor_returns_nothing() { + // The mirror image of `internetarchive`: this plugin can ONLY work from + // an id anchor, because you cannot search YouTube for a UUID and an + // unverified match is the one output this tier must never produce. + let p = YouTubePlugin::default(); + let q = GatewayQuery { + raw_text: "daft punk".to_string(), + free_text: "daft punk".to_string(), + filters: BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + }; + assert!(p.handle_query(&q, 20).await.expect("ok").is_empty()); + } + + fn tref(video: &str, delta: i64, key: &str) -> TrackRef { + TrackRef { + video_id: video.to_string(), + title: video.to_string(), + match_key: key.to_string(), + delta, + } + } + + fn pick(recording: &str, refs: Vec) -> TrackPick { + TrackPick { + recording_mbid: recording.to_string(), + track: musicbrainz_feeder::client::Track { + title: recording.to_string(), + ..Default::default() + }, + refs, + } + } + + /// §7.5.1 — the collision rule, with the study's own measured case. + /// + /// Burial's *Archangel* and *Near Dark* both resolved to `VVa_aCSA9Jc` at + /// EXACT: same channel, both durations inside tolerance. They are different + /// recordings, so at least one is wrong. The tighter delta keeps it and the + /// other drops the reference entirely — emitting both would put the same + /// audio under two names. + #[test] + fn two_recordings_never_share_one_video() { + let mut picks = vec![ + pick("archangel", vec![tref("VVa_aCSA9Jc", 7, "mb:youtube-music")]), + pick("near-dark", vec![tref("VVa_aCSA9Jc", 1, "mb:youtube-music")]), + ]; + assign_references(&mut picks); + assert!(picks[0].refs.is_empty(), "the looser claim loses the video"); + assert_eq!(picks[1].refs.len(), 1, "the tighter delta keeps it"); + } + + /// A collision must not cost a track its *other*, uncontested references. + #[test] + fn a_collision_only_drops_the_contested_reference() { + let mut picks = vec![ + pick("a", vec![tref("shared", 9, "mb:youtube"), tref("own-a", 2, "mb:youtube")]), + pick("b", vec![tref("shared", 1, "mb:youtube")]), + ]; + assign_references(&mut picks); + assert_eq!(picks[0].refs.len(), 1); + assert_eq!(picks[0].refs[0].video_id, "own-a"); + assert_eq!(picks[1].refs[0].video_id, "shared"); + } + + /// §3 — several ranked references per recording is the rot mitigation. + /// Uncontested ones all survive, in rank order. + #[test] + fn a_recording_keeps_several_ranked_references() { + let mut picks = vec![pick( + "rec", + vec![ + tref("best", 0, "mb:youtube-music"), + tref("second", 3, "mb:youtube"), + tref("third", 8, "piped:music-artists"), + ], + )]; + assign_references(&mut picks); + assert_eq!(picks[0].refs.len(), 3); + assert_eq!(picks[0].refs[0].video_id, "best"); + } + + /// The tie-break behind that order: an equal duration delta is settled by + /// how authoritative the channel key is, never arbitrarily. + #[test] + fn channel_key_strength_breaks_a_duration_tie() { + assert!(key_priority("mb:youtube-music") < key_priority("mb:youtube")); + assert!(key_priority("mb:youtube") < key_priority("piped:music-artists")); + } + + #[test] + fn the_cover_url_is_hqdefault_not_maxres() { + // `maxresdefault` 404s on anything not uploaded in HD, and that 404 is + // SILENT: the gate drops the record with no error anywhere. + let url = format!("{YT_THUMB_BASE}/4D7u5KF7SP8/hqdefault.jpg"); + assert!(url.ends_with("/hqdefault.jpg")); + assert!(meta_feeder_sdk::hash::artwork_locator(&url).is_some()); + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]