diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..afad20f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/usenet-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/usenet-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1de5659 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..5459dc6 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2928 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.9", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "bitflags", + "libc", + "plain", + "redox_syscall 0.9.0", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.19", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.2", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.19", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.9", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlx" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +dependencies = [ + "base64", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.5", + "hashlink", + "indexmap", + "log", + "memchr", + "once_cell", + "percent-encoding", + "rustls", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.19", + "tokio", + "tokio-stream", + "tracing", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.119", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.119", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand 0.8.7", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.19", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +dependencies = [ + "atoi", + "base64", + "bitflags", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand 0.8.7", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.19", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.19", + "tracing", + "url", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "usenet-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "chrono", + "flate2", + "futures", + "meta-feeder-sdk", + "quick-xml", + "regex", + "reqwest", + "serde", + "serde_json", + "sqlx", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.9", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" +dependencies = [ + "libredox", + "wasite", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[patch.unused]] +name = "core2" +version = "0.4.0" +source = "git+https://github.com/bbqsrc/core2?rev=545e84b#545e84bcb0f235b12e21351e0c69767958efe2a7" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..023569c --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,18 @@ +[workspace] +resolver = "2" +members = ["crates/*", "feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-usenet" + +# --------------------------------------------------------------------------- +# Safety net for the yanked transitive `core2` dep, kept identical to the +# meta-gateway / meta-feeder-* workspaces. Harmless (an unused-patch warning) +# if the dep graph doesn't pull it; required if it does. Remove once +# multihash-codetable drops the core2 dependency upstream. +# --------------------------------------------------------------------------- +[patch.crates-io] +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84b" } diff --git a/README.md b/README.md index 42f4fed..2f084a5 100644 --- a/README.md +++ b/README.md @@ -1 +1,102 @@ -# meta-feeder-usenet \ No newline at end of file +# meta-feeder-usenet + +Usenet feeder sidecar for the MetaMesh gateway tier. Bridges a Usenet header +scan **we run ourselves** (an [nntmux](https://github.com/NNTmux/newznab-tmux) +sidecar) into the MetaMesh network. + +## Why this exists + +`meta-feeder-indexer` proxies an **external** Newznab indexer and mints an +`nzb-release` (`0x1005`) cid that embeds *that indexer's host*. Only a peer +holding that indexer's credential can ever redeem it — so the metadata half of +Usenet is neither self-owned nor portable. + +Scanning Usenet ourselves gives us the article Message-IDs for free at collation +time (no grab, no scrape). That lets this feeder mint a **portable** +`nzb-posting` (`0x1003`) cid — a digest over the Message-ID set, embedding no +host — which **any peer with a plain NNTP provider can redeem, with no indexer +credential anywhere in the path**. + +This feeder **adds** a Usenet metadata source alongside `meta-feeder-indexer`. +It replaces nothing; a gateway query fans out to both. + +Design record: +`meta-gateway/docs/others/self-hosted-usenet-indexer-study.md`. + +## Shape + +``` +[nntmux sidecar: PHP + MariaDB + Redis] [usenet-feeder: Rust, FeederPlugin] + header-scan groups (few conns) ──► reads `releases` (searchname/size/guid) + collate → releases ──► gunzips //.nzb.gz + writes .nzb.gz ──► mints 0x1003 from the Message-ID set + (CBP purged; the gz is the only ──► emits the record + a relative + durable home of the msg-ids) `manifest_url` onto its own /blob route + │ + ▼ + gateway core: seeds the .nzb as an ORDINARY sha2-256 IPFS cid, publishes it + as a fileType=nzb meta-core record, rewrites the field to that cid + (the same path posters take — `remote_feeder.rs` SEEDABLE_FIELDS) + │ + ▼ + meta-share: resolve `manifest` cid (blockstore / bitswap / ANY holder) → + NzbManifest::from_nzb_xml → BODY-fetch each Message-ID from ANY plain NNTP + provider → materialise → play +``` + +The sidecar is **persistent cache, cheap to lose** — everything in it is +re-derivable by re-scanning. Same pattern as the Tribler sidecar: stateful +upstream, but the gateway still sees a stateless HTTP feeder. + +Like every feeder this one is **meta-core-free and blockstore-free**: it finds +records and serves bytes; the core owns hashing-into-the-blockstore, the +meta-core store-back, and the libp2p wire (gateway invariant 10). + +## Configuration + +Operator config lives on the **feeder**, not the gateway (invariant 12). Set it +on the plugin's config page in the gateway dashboard; the env vars below are a +**first-boot seed only** — the persisted `config.json` wins on the next restart, +and there is no hot reload. + +| Field | Env seed | What | +|---|---|---| +| `db_url` | `NNTMUX_DB_URL` | MariaDB DSN of nntmux's catalog, e.g. `mysql://nntmux:secret@nntmux-db:3306/nntmux`. Blank → the upstream soft-skips and the plugin reports Degraded. | +| `nzb_root` | `NNTMUX_NZB_PATH` | nntmux's NZB directory **as mounted into this container**. Must be the same volume nntmux's `PATH_TO_NZBS` points at, or every release resolves to `NotFound`. | + +Infra env: `META_FEEDER_HTTP_LISTEN` (default `0.0.0.0:8080`), +`META_FEEDER_STATE_DIR` (default `/data/meta-feeder`), `RUST_LOG`. + +Scan policy — which groups, backfill depth, scan interval, connection budget — +is the **gateway maintainer's** call and is configured on the nntmux sidecar. +Keep the connection budget conservative: the scanner and playback draw on the +same provider slots, so a greedy value does not merely over-scan, it stalls +playback mid-stream. + +## Build + +```bash +cargo build --release --bin usenet-feeder +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings + +docker build -f feeder-plugin/usenet-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-usenet:dev . +``` + +`crates/meta-feeder-sdk/` is a **vendored copy** of the SDK, mirrored from +`meta-gateway`. Independent submodules cannot share a cargo path-dep, so keep it +in step when the SDK changes — in particular `hash.rs`'s +`compute_nzb_posting_cid`, whose normalisation rule is the dedup contract and is +mirrored again on the meta-share side. + +## Gotchas + +- **An empty catalog is the visible symptom of both nntmux silent-zero traps**: + an empty `PATH_TO_NZBS` (set-but-empty beats the default, so `createNZBs()` + aborts and you get 0 releases with no error), and seeded groups left + `active=0`. The plugin logs a loud warning when `releases` is empty at boot — + believe it. +- **The Message-IDs are not in the database.** nntmux purges + collections/binaries/parts right after writing each NZB, so the on-disk + `.nzb.gz` is the only durable home. A wrong `nzb_root` therefore fails at + `compute_outcomes`, not at query time. diff --git a/crates/meta-feeder-sdk/Cargo.toml b/crates/meta-feeder-sdk/Cargo.toml new file mode 100644 index 0000000..bbe98ff --- /dev/null +++ b/crates/meta-feeder-sdk/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "meta-feeder-sdk" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Source-agnostic foundation shared by the gateway core and every feeder +# sidecar binary: the `FeederPlugin` trait, the content-addressing hash +# families, the per-plugin redb cache, the shared HTTP/cache helpers, the +# structured-query evaluator, and the axum `serve_feeders` harness that wraps +# any set of `FeederPlugin`s in the feeder HTTP contract. +# +# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes; +# the gateway core owns hashing-into-the-blockstore, the meta-core store-back, +# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT +# of here so feeders that don't need them don't pay for them. + +[dependencies] +# Async runtime + HTTP server (the serve_feeders harness) +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +hyper = { version = "1", features = ["full"] } +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" +base64 = "0.22" + +# Utilities +futures = "0.3" +async-trait = "0.1" +anyhow = "1" +thiserror = "1" +tracing = "0.1" +sha2 = "0.10" +bytes = "1" +# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). +regex = "1" + +# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see +# cache.rs. Pure-Rust single-process embedded store. +redb = "2" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/crates/meta-feeder-sdk/src/cache.rs b/crates/meta-feeder-sdk/src/cache.rs new file mode 100644 index 0000000..f22e99e --- /dev/null +++ b/crates/meta-feeder-sdk/src/cache.rs @@ -0,0 +1,337 @@ +//! Per-plugin midhash cache backed by redb. +//! +//! Each plugin gets its own redb file at +//! `/gateway//cache.redb`. Keys are upstream +//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the +//! `Midhash` strings the plugin computed by fetching + hashing the upstream +//! file. +//! +//! The schema is identical to meta-share v1's so old `.redb` files copy +//! across. redb is synchronous and sub-millisecond for small ops, so calls +//! run inline from async handlers without `spawn_blocking`. + +use std::path::Path; +use std::sync::Arc; + +use redb::{Database, ReadableTableMetadata, TableDefinition}; + +pub const CACHE_FILENAME: &str = "cache.redb"; + +const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash"); +const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs"); +const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid"); +const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec"); +const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid"); +/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload +/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the +/// hash families differ: `midhash` is the synthetic midhash256-from-infohash +/// fallback; `fullhash` records "we actually downloaded the bytes and this is +/// the IPFS CIDv1 over them." +const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash"); +/// Enumerated torrent file list (JSON `Vec`) keyed by the +/// torrent's `record_id` (infohash). +const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist"); +/// Cached TMDB *search* results keyed by a stable lookup key. The value is the +/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache +/// sentinel. +const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search"); +/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`. +const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_tvdetails"); +/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`. +const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids"); +/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`. +const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_moviedetails"); +/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized +/// free-text query. +const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_principal_topn"); +/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's +/// `record_id` (infohash). Value is the JSON encoding of a `Vec`. +const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles"); +/// OpenSubtitles search results keyed by `"\x01"`. Value is the +/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel. +const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles"); + +/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared +/// via `Arc`). +#[derive(Clone)] +pub struct MidhashCache { + db: Arc, +} + +/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb +/// table. +macro_rules! str_table_accessors { + ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => { + $(#[$gmeta])* + pub fn $get(&self, key: &str) -> Result, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table($table)?; + Ok(table.get(key)?.map(|v| v.value().to_string())) + } + + $(#[$pmeta])* + pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table($table)?; + table.insert(key, value)?; + } + tx.commit()?; + Ok(()) + } + }; +} + +impl MidhashCache { + pub fn open(cache_dir: &Path) -> Result { + let path = cache_dir.join(CACHE_FILENAME); + let db = Database::create(&path)?; + { + let tx = db.begin_write()?; + tx.open_table(MIDHASH_TABLE)?; + tx.open_table(BLOBS_TABLE)?; + tx.open_table(COVER_CID_TABLE)?; + tx.open_table(BIBREC_TABLE)?; + tx.open_table(PREVIEW_CID_TABLE)?; + tx.open_table(FULLHASH_TABLE)?; + tx.open_table(FILELIST_TABLE)?; + tx.open_table(TMDB_SEARCH_TABLE)?; + tx.open_table(TMDB_TVDETAILS_TABLE)?; + tx.open_table(TMDB_EXTIDS_TABLE)?; + tx.open_table(TMDB_MOVIEDETAILS_TABLE)?; + tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?; + tx.open_table(SUBTITLES_TABLE)?; + tx.open_table(OPENSUBTITLES_TABLE)?; + tx.commit()?; + } + Ok(Self { db: Arc::new(db) }) + } + + str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE); + + pub fn entry_count(&self) -> Result { + let tx = self.db.begin_read()?; + let table = tx.open_table(MIDHASH_TABLE)?; + Ok(table.len()?) + } + + pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BLOBS_TABLE)?; + Ok(table.get(cid)?.map(|v| v.value().to_vec())) + } + + pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BLOBS_TABLE)?; + table.insert(cid, bytes)?; + } + tx.commit()?; + Ok(()) + } + + str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE); + + str_table_accessors!( + /// Read a previously-stored full-file IPFS cid for `record_id`. + get_fullhash, + /// Record the IPFS cid produced by a successful full fetch. + put_fullhash, + FULLHASH_TABLE + ); + + str_table_accessors!( + /// Read the cached torrent file list (JSON) for `record_id`. + get_filelist, + /// Cache the enumerated torrent file list (JSON `Vec`). + put_filelist, + FILELIST_TABLE + ); + + str_table_accessors!( + /// Read the cached subtitle linkage (JSON `Vec`). + get_subtitles, + /// Cache the discovered subtitle linkage (JSON `Vec`). + put_subtitles, + SUBTITLES_TABLE + ); + + str_table_accessors!( + /// Read a cached OpenSubtitles lookup by `"\x01"`. + get_opensubtitles, + /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel). + put_opensubtitles, + OPENSUBTITLES_TABLE + ); + + str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE); + + str_table_accessors!( + /// Read a cached TMDB search result by lookup key. + get_tmdb_search, + /// Cache a TMDB search result under `key`. + put_tmdb_search, + TMDB_SEARCH_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB TV-details JSON by `tmdbid`. + get_tmdb_tvdetails, + /// Cache TMDB TV-details JSON under `tmdbid`. + put_tmdb_tvdetails, + TMDB_TVDETAILS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB external-ids JSON by `tmdbid`. + get_tmdb_extids, + /// Cache TMDB external-ids JSON under `tmdbid`. + put_tmdb_extids, + TMDB_EXTIDS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB movie-details JSON by `tmdbid`. + get_tmdb_moviedetails, + /// Cache TMDB movie-details JSON under `tmdbid`. + put_tmdb_moviedetails, + TMDB_MOVIEDETAILS_TABLE + ); + + str_table_accessors!( + /// Read the cached ranked anchor list by normalized query key. + get_tmdb_principal_topn, + /// Cache the ranked anchor list under the normalized query key. + put_tmdb_principal_topn, + TMDB_PRINCIPAL_TOPN_TABLE + ); + + pub fn get_bibrec( + &self, + record_id: &str, + ) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BIBREC_TABLE)?; + let Some(v) = table.get(record_id)? else { + return Ok(None); + }; + let s = v.value(); + Ok(serde_json::from_str(s).ok()) + } + + pub fn put_bibrec( + &self, + record_id: &str, + fields: &std::collections::BTreeMap, + ) -> Result<(), redb::Error> { + let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()); + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BIBREC_TABLE)?; + table.insert(record_id, json.as_str())?; + } + tx.commit()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_cache_dir() -> tempfile::TempDir { + tempfile::tempdir().expect("tempdir") + } + + #[test] + fn put_then_get_returns_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None); + cache.put_midhash("10.1038/x", "bafyXYZ").expect("put"); + assert_eq!( + cache.get_midhash("10.1038/x").unwrap().as_deref(), + Some("bafyXYZ") + ); + } + + #[test] + fn put_overwrites_existing_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + cache.put_midhash("k", "v1").unwrap(); + cache.put_midhash("k", "v2").unwrap(); + assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2")); + } + + #[test] + fn values_persist_across_reopens() { + let dir = tmp_cache_dir(); + { + let cache = MidhashCache::open(dir.path()).expect("open 1"); + cache.put_midhash("k", "persistent").unwrap(); + } + let cache = MidhashCache::open(dir.path()).expect("open 2"); + assert_eq!( + cache.get_midhash("k").unwrap().as_deref(), + Some("persistent") + ); + } + + #[test] + fn tmdb_search_cache_distinguishes_miss_from_absent() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None); + cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap(); + assert_eq!( + cache + .get_tmdb_search("m\u{1}naruto\u{1}") + .unwrap() + .as_deref(), + Some("null") + ); + cache + .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}") + .unwrap(); + drop(cache); + let cache = MidhashCache::open(dir.path()).expect("reopen"); + assert_eq!( + cache + .get_tmdb_search("t\u{1}spy x family\u{1}2019") + .unwrap() + .as_deref(), + Some("{\"tmdbid\":120089}") + ); + } + + #[test] + fn tmdb_tvdetails_cache_roundtrips() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None); + cache + .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}") + .unwrap(); + assert_eq!( + cache.get_tmdb_tvdetails("120089").unwrap().as_deref(), + Some("{\"number_of_seasons\":2}") + ); + } + + #[test] + fn entry_count_reflects_inserts() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.entry_count().unwrap(), 0); + cache.put_midhash("a", "x").unwrap(); + cache.put_midhash("b", "y").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + cache.put_midhash("a", "x2").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + } +} diff --git a/crates/meta-feeder-sdk/src/common.rs b/crates/meta-feeder-sdk/src/common.rs new file mode 100644 index 0000000..c7a09f5 --- /dev/null +++ b/crates/meta-feeder-sdk/src/common.rs @@ -0,0 +1,288 @@ +//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`, +//! `pubmed`, `wikicommons`, `giphy`, `scihub`). +//! +//! These plugins are structurally near-identical: build a reqwest client, map +//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose +//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding +//! lives here; the genuinely-different bodies stay in the plugin modules. + +use std::path::Path; +use std::time::Duration; + +use tracing::{debug, warn}; + +use crate::cache::MidhashCache; +use crate::plugin::{ConfigError, HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, GatewayError, Hash}; + +/// Build the shared reqwest client every source plugin uses. +pub fn build_http_client( + timeout_secs: u64, + user_agent: &str, + redirect: Option, +) -> reqwest::Client { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(timeout_secs)) + .user_agent(user_agent); + if let Some(policy) = redirect { + builder = builder.redirect(policy); + } + builder + .build() + .expect("reqwest client build is infallible with rustls-tls") +} + +/// Map a non-success HTTP status to a [`GatewayError`]. +/// +/// - `404` → [`GatewayError::NotFound`] +/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s) +/// - `5xx` → [`GatewayError::Transient`] +/// - anything else → [`GatewayError::Permanent`] +pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> { + let status = resp.status(); + if status.is_success() { + return Ok(()); + } + let code = status.as_u16(); + if code == 404 { + return Err(GatewayError::NotFound); + } + if code == 429 { + let retry_after = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(60); + return Err(GatewayError::RateLimited { + retry_after_s: retry_after, + }); + } + if status.is_server_error() { + return Err(GatewayError::Transient(format!( + "upstream {} returned {status}", + resp.url() + ))); + } + Err(GatewayError::Permanent(format!( + "upstream {} returned {status}", + resp.url() + ))) +} + +/// Open the per-plugin redb midhash cache, wrapping the error with the plugin +/// name for a clean startup diagnostic. +pub fn open_midhash_cache( + cache_dir: &Path, + plugin: &'static str, +) -> Result { + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin, + source: anyhow::anyhow!("open redb cache: {e}"), + }) +} + +/// Cache-hit fast path for single-outcome plugins. +/// +/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record — +/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a +/// cache miss; the caller proceeds to fetch + hash. +pub fn cached_outcome( + cache: &MidhashCache, + record_id: &str, + upstream_id: &'static str, +) -> Result>, GatewayError> { + let Some(cached) = cache + .get_midhash(record_id) + .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))? + else { + return Ok(None); + }; + debug!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + "hash cache hit" + ); + Ok(Some(vec![HashOutcome { + hash: Hash(cached), + hash_kind: HashKind::Sha2_256, + bytes: None, + record: None, + file_extension: None, + }])) +} + +/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is +/// non-fatal, so it is logged and swallowed. +pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) { + if let Err(e) = cache.put_midhash(record_id, cid) { + warn!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + error = %e, + "hash cache put failed (non-fatal)" + ); + } +} + +/// Resolve the per-plugin cache handle, or produce the standard "not +/// configured" internal error. +pub fn require_cache<'a>( + cache: Option<&'a MidhashCache>, + plugin: &str, +) -> Result<&'a MidhashCache, GatewayError> { + cache.ok_or_else(|| { + GatewayError::Internal(anyhow::anyhow!( + "{plugin} plugin not configured (configure() never called)" + )) + }) +} + +/// Build the single-element outcome vec a single-outcome plugin returns on a +/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID, +/// and an optional file extension. +pub fn single_outcome( + cid: String, + bytes: bytes::Bytes, + record: DiscoveryRecord, + file_extension: Option, +) -> Vec { + vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record: Some(record), + file_extension, + }] +} + +/// Percent-encode a query-string value using the unreserved set +/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space. +pub fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.as_bytes() { + match *b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*b as char); + } + b' ' => out.push('+'), + other => out.push_str(&format!("%{:02X}", other)), + } + } + out +} + +/// GET `url` and return the body bytes, mapping transport + status errors to +/// [`GatewayError`]. +pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.bytes() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Like [`fetch_bytes`] but decodes the body as UTF-8 text. +pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.text() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Decode a v1 BitTorrent infohash string into its raw 20 bytes. +/// +/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648, +/// case-insensitive). Returns `None` for anything else. +pub fn decode_infohash(s: &str) -> Option<[u8; 20]> { + let bytes = s.as_bytes(); + let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) { + let mut out = Vec::with_capacity(20); + for pair in bytes.chunks(2) { + out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?); + } + out + } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) { + decode_base32_20(bytes)? + } else { + return None; + }; + raw.try_into().ok() +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20 +/// bytes they encode. +fn decode_base32_20(chars: &[u8]) -> Option> { + let mut out = Vec::with_capacity(20); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &c in chars { + let val = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a', + b'2'..=b'7' => c - b'2' + 26, + _ => return None, + } as u64; + buffer = (buffer << 5) | val; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push((buffer >> bits) as u8); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_infohash_hex_roundtrips() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let raw = decode_infohash(hex).expect("hex decodes"); + assert_eq!(raw[0], 0x01); + assert_eq!(raw[19], 0x67); + } + + #[test] + fn decode_infohash_is_case_insensitive_hex() { + let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd"); + let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD"); + assert!(lower.is_some()); + assert_eq!(lower, upper); + } + + #[test] + fn decode_infohash_accepts_base32() { + let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U"); + assert!(raw.is_some()); + } + + #[test] + fn decode_infohash_rejects_wrong_shape() { + assert!(decode_infohash("tooshort").is_none()); + assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none()); + assert!(decode_infohash("").is_none()); + } +} diff --git a/crates/meta-feeder-sdk/src/config.rs b/crates/meta-feeder-sdk/src/config.rs new file mode 100644 index 0000000..c4af5d6 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config.rs @@ -0,0 +1,312 @@ +//! Self-describing per-plugin configuration: schema, redaction, merge, and the +//! generic schema-driven HTML page the feeder serves at `/config`. +//! +//! The point of this module is that **the gateway carries no per-plugin config +//! knowledge** and **the SDK carries no per-plugin field names**. A plugin +//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema); +//! everything here (redaction of secrets, merge-on-save, and the rendered form) +//! is driven entirely by that schema. Add a feeder → it shows up with a working +//! config form and zero UI/gateway changes. +//! +//! ## Persistence & precedence +//! +//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to +//! `configure()`). The plugin reads **file-or-env** at startup: env is the +//! first-boot seed and stays authoritative until the operator saves through the +//! UI, at which point `config.json` exists and wins. There is no runtime reload +//! — a save returns `restart_required: true` (mirrors the gateway's +//! `gateway-config.json` posture). +//! +//! ## Secrets +//! +//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to +//! the page (redacted to a `_set: bool`), and a blank incoming value on +//! save means "keep the stored secret". + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A plugin's full config surface — an ordered list of fields. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ConfigSchema { + pub fields: Vec, +} + +impl ConfigSchema { + /// Convenience: `true` when the plugin declares no configurable fields (the + /// page then renders a "no configuration required" notice). + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } +} + +/// One configurable field. `record_array` fields nest `fields` for the +/// per-record sub-schema (e.g. torznab's `indexers`). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ConfigField { + /// JSON key in the config object. + pub key: String, + /// Human label for the form. + pub label: String, + #[serde(rename = "type")] + pub kind: FieldKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub help: Option, + #[serde(default)] + pub required: bool, + /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated + /// as the record identifier for secret back-fill on merge (e.g. `name`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +impl ConfigField { + pub fn text(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Text) + } + pub fn secret(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Secret) + } + pub fn bool(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Bool) + } + pub fn number(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Number) + } + pub fn list(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::List) + } + pub fn record_array(key: &str, label: &str, fields: Vec) -> Self { + let mut f = Self::new(key, label, FieldKind::RecordArray); + f.fields = Some(fields); + f + } + fn new(key: &str, label: &str, kind: FieldKind) -> Self { + Self { + key: key.to_string(), + label: label.to_string(), + kind, + help: None, + required: false, + fields: None, + } + } + pub fn with_help(mut self, help: &str) -> Self { + self.help = Some(help.to_string()); + self + } + pub fn required(mut self) -> Self { + self.required = true; + self + } +} + +/// Field widget / value type. Drives both rendering and the secret/merge rules. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FieldKind { + /// Free text → JSON string. + Text, + /// Write-only secret → JSON string; redacted on read, "blank keeps" on save. + Secret, + /// Checkbox → JSON bool. + Bool, + /// Numeric input → JSON number. + Number, + /// Comma/line list → JSON array of strings. + List, + /// Repeatable record → JSON array of objects keyed by nested `fields`. + RecordArray, +} + +fn is_blank(v: Option<&Value>) -> bool { + match v { + None | Some(Value::Null) => true, + Some(Value::String(s)) => s.is_empty(), + _ => false, + } +} + +/// Produce a UI-safe view of `values`: secret fields are dropped and replaced +/// with a `_set` boolean; everything else passes through. Recurses into +/// `record_array` records for secret sub-fields. +pub fn redact(values: &Value, schema: &ConfigSchema) -> Value { + let src = values.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&src, &schema.fields)) +} + +fn redact_object(src: &Map, fields: &[ConfigField]) -> Map { + let mut out = Map::new(); + for f in fields { + match f.kind { + FieldKind::Secret => { + let set = !is_blank(src.get(&f.key)); + out.insert(format!("{}_set", f.key), Value::Bool(set)); + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let arr = src + .get(&f.key) + .and_then(Value::as_array) + .map(|recs| { + recs.iter() + .map(|r| { + let m = r.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&m, subs)) + }) + .collect::>() + }) + .unwrap_or_default(); + out.insert(f.key.clone(), Value::Array(arr)); + } + _ => { + if let Some(v) = src.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// Merge `incoming` (from the UI) onto `base` (the stored/effective config), +/// schema-aware. Returns the full, UNREDACTED object to persist: +/// - secret: blank/absent incoming keeps `base`; non-blank overwrites. +/// - text/number/bool/list: present in incoming overwrites; absent keeps base. +/// - record_array: incoming array replaces base, but blank secret sub-fields are +/// back-filled from the base record matched by the first sub-field (identifier). +pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value { + let base_obj = base.as_object().cloned().unwrap_or_default(); + let inc_obj = incoming.as_object().cloned().unwrap_or_default(); + Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields)) +} + +fn merge_object( + base: &Map, + inc: &Map, + fields: &[ConfigField], +) -> Map { + let mut out = base.clone(); + for f in fields { + match f.kind { + FieldKind::Secret => { + if !is_blank(inc.get(&f.key)) { + out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap()); + } + // blank/absent → keep base (no-op) + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let id_key = subs.first().map(|s| s.key.clone()); + let base_recs = base.get(&f.key).and_then(Value::as_array).cloned(); + let inc_recs = match inc.get(&f.key).and_then(Value::as_array) { + Some(r) => r.clone(), + None => continue, // absent → keep base array untouched + }; + let merged: Vec = inc_recs + .iter() + .map(|rec| { + let rec_obj = rec.as_object().cloned().unwrap_or_default(); + // Find the matching base record by identifier sub-field. + let base_match = match (&id_key, base_recs.as_ref()) { + (Some(idk), Some(brs)) => brs + .iter() + .find(|br| br.get(idk) == rec_obj.get(idk)) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(), + _ => Map::new(), + }; + Value::Object(merge_object(&base_match, &rec_obj, subs)) + }) + .collect(); + out.insert(f.key.clone(), Value::Array(merged)); + } + _ => { + if let Some(v) = inc.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// The generic, schema-driven config page served at `GET /config`. Self-contained +/// (inline CSS + JS, no external assets) so it loads identically when hit +/// directly on the feeder OR reverse-proxied through the gateway. All API calls +/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant. +pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html"); + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn torznab_like() -> ConfigSchema { + ConfigSchema { + fields: vec![ + ConfigField::secret("tmdb_token", "TMDB token"), + ConfigField::bool("resolve_torrent_files", "Resolve .torrent"), + ConfigField::record_array( + "indexers", + "Indexers", + vec![ + ConfigField::text("name", "Name"), + ConfigField::secret("api_key", "API key"), + ], + ), + ], + } + } + + #[test] + fn redact_hides_secrets_and_flags_set() { + let v = json!({ + "tmdb_token": "abc", + "resolve_torrent_files": true, + "indexers": [{"name":"prowlarr","api_key":"k"}] + }); + let r = redact(&v, &torznab_like()); + assert_eq!(r["tmdb_token_set"], json!(true)); + assert!(r.get("tmdb_token").is_none()); + assert_eq!(r["resolve_torrent_files"], json!(true)); + assert_eq!(r["indexers"][0]["name"], json!("prowlarr")); + assert_eq!(r["indexers"][0]["api_key_set"], json!(true)); + assert!(r["indexers"][0].get("api_key").is_none()); + } + + #[test] + fn merge_blank_secret_keeps_base() { + let base = json!({"tmdb_token": "keepme"}); + let inc = json!({"tmdb_token": ""}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("keepme")); + } + + #[test] + fn merge_nonblank_secret_overwrites() { + let base = json!({"tmdb_token": "old"}); + let inc = json!({"tmdb_token": "new"}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("new")); + } + + #[test] + fn merge_record_array_backfills_blank_secret_by_identifier() { + let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]}); + // UI re-submits the indexer with a blank api_key → keep the stored one. + let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["indexers"][0]["api_key"], json!("secret")); + assert_eq!(m["indexers"][0]["name"], json!("prowlarr")); + } + + #[test] + fn merge_bool_overwrites_when_present() { + let base = json!({"resolve_torrent_files": true}); + let inc = json!({"resolve_torrent_files": false}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["resolve_torrent_files"], json!(false)); + } +} diff --git a/crates/meta-feeder-sdk/src/config_page.html b/crates/meta-feeder-sdk/src/config_page.html new file mode 100644 index 0000000..17d9d74 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config_page.html @@ -0,0 +1,268 @@ + + + + + +feeder config + + + +

Plugin configuration

+

Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked * are required.

+ +
+ + + + + diff --git a/crates/meta-feeder-sdk/src/enrich.rs b/crates/meta-feeder-sdk/src/enrich.rs new file mode 100644 index 0000000..b55a9f8 --- /dev/null +++ b/crates/meta-feeder-sdk/src/enrich.rs @@ -0,0 +1,614 @@ +//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline. +//! +//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base +//! record, then drives the **reused meta-sort enrichment plugins** +//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each +//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to +//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches +//! enrichment — full separation of concerns. +//! +//! ## Ordering — why we poll +//! +//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from +//! the record), and those are produced by filename-parser. So the pipeline is +//! **filename-parser → tmdb**. The plugin `/process` contract is *async* +//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after +//! kicking filename-parser we **poll meta-core** until `originalTitle` appears +//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps +//! the feeder stateless — no callback-correlation bookkeeping. The plugins' +//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`. +//! +//! The whole pipeline runs in a spawned background task so it never adds latency +//! to the libp2p resolve path; the meta-share ingester propagates the enriched +//! record on its next poll (~eventually consistent). + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use tracing::{debug, warn}; + +use crate::meta_core::{self, Provenance}; +use crate::types::DiscoveryRecord; + +/// Operator-supplied enrichment wiring, parsed once from the feeder env. +#[derive(Clone, Debug)] +pub struct EnrichmentConfig { + /// meta-core root the feeder self-publishes to + hands to the plugins. + pub meta_core_url: String, + /// `metamesh-plugin-filename-parser` base URL (optional). + pub filename_parser_url: Option, + /// `metamesh-plugin-tmdb` base URL (optional). + pub tmdb_url: Option, + /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`. + pub tmdb_token: Option, + /// TMDB metadata language (default `en-US`). + pub tmdb_language: String, + /// Stable per-deployment id stamped into `provenance.gatewayPeer`. + pub gateway_peer: String, + /// URL the plugins POST their completion callback to (the feeder's own + /// no-op `/enrich/callback`). Polling is the real completion signal; this + /// only keeps the plugins from logging a failed-callback warning. + pub callback_url: String, + /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the + /// pipeline drives subtitle fetching after tmdb resolves the ids — the + /// single reference implementation of OpenSubtitles fetching (the gateway + /// no longer carries its own). + pub opensubtitles_url: Option, + /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`. + pub opensubtitles_api_key: Option, + /// OpenSubtitles account username (required by the plugin for downloads). + pub opensubtitles_username: Option, + /// OpenSubtitles account password. + pub opensubtitles_password: Option, + /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). + pub opensubtitles_languages: String, +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the +/// app-compose convention `-feeder` → `-:8080` +/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets +/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit +/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns +/// `None` when `HOSTNAME` is unset or not in `-feeder` form, in which case +/// the caller falls back to the env seed (then soft-skips that plugin). +fn derive_sidecar_url(plugin: &str) -> Option { + let host = env_nonempty("HOSTNAME")?; + let prefix = host.strip_suffix("-feeder")?; + Some(format!("http://{prefix}-{plugin}:8080")) +} + +impl EnrichmentConfig { + /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset — + /// the **bundled-feeder** path (records returned over HTTP, the gateway + /// dispatcher stores them). Self-publishing feeders that source meta-core from + /// their own dashboard config use [`from_meta_core`](Self::from_meta_core). + pub fn from_env() -> Option { + Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?)) + } + + /// Build with an explicit meta-core root — sourced from the feeder's own + /// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The + /// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the + /// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest + /// keep their env seeds / defaults. Secrets (tmdb token, language) are + /// overlaid by the caller from the per-feeder config. + pub fn from_meta_core(meta_core_url: String) -> Self { + EnrichmentConfig { + meta_core_url, + filename_parser_url: env_nonempty("FILENAME_PARSER_URL") + .or_else(|| derive_sidecar_url("filename-parser")), + tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")), + tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")), + tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()), + gateway_peer: env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()), + callback_url: env_nonempty("META_FEEDER_CALLBACK_URL") + .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()), + opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"), + opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"), + opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"), + opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), + opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") + .unwrap_or_else(|| "en".to_string()), + } + } +} + +/// How long to wait for filename-parser to merge `originalTitle` before handing +/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous +/// ceiling, not an expected wait. +const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5); +/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the +/// sibling files. tmdb hits the network, so a longer ceiling than filename. +const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Show-level TMDB fields that are **fanned** from the primary file to every +/// sibling file in a multi-file (season-pack) torrent — they're identical for +/// every episode of a show, so we resolve tmdb **once** and copy these across +/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which +/// gateway BT records don't carry, so without this each file would re-hit TMDB). +/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`, +/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file. +const ANCHOR_EXACT: &[&str] = &[ + "tmdbid", + "imdbid", + "poster", + "backdrop", + "rating", + "releasedate", + "studio", + "genres", + "tags", +]; +const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"]; + +/// One file (or pack) to enrich: its CID, the filename to parse, and the base +/// fields already written to meta-core (handed to the plugins as `existingMeta`). +#[derive(Clone, Debug)] +pub struct EnrichTarget { + pub cid: String, + pub file_name: String, + pub base: BTreeMap, +} + +/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to +/// clone (just a reqwest client + an `Arc`), so it can be moved into a +/// spawned per-resolve task. +#[derive(Clone)] +pub struct Enricher { + http: reqwest::Client, + cfg: Arc, +} + +#[derive(Serialize)] +struct ProcessRequest<'a> { + #[serde(rename = "taskId")] + task_id: String, + cid: &'a str, + #[serde(rename = "filePath")] + file_path: &'a str, + #[serde(rename = "callbackUrl")] + callback_url: &'a str, + #[serde(rename = "metaCoreUrl")] + meta_core_url: &'a str, + #[serde(rename = "existingMeta")] + existing_meta: &'a BTreeMap, +} + +impl Enricher { + pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self { + Enricher { + http, + cfg: Arc::new(cfg), + } + } + + pub fn meta_core_url(&self) -> &str { + &self.cfg.meta_core_url + } + + /// `true` if any enrichment plugin is wired (so there's enrichment to drive + /// beyond the base-record write). + pub fn has_plugins(&self) -> bool { + self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some() + } + + /// Write the feeder's base record into meta-core (merge/upsert) under `cid`. + /// Identical shape to the gateway dispatcher's write (provenance + the + /// `cids/` key-set collapse). + pub async fn write_base_record( + &self, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now( + &record.upstream_id, + &record.record_id, + &self.cfg.gateway_peer, + ); + let body = meta_core::build_metadata_body(record, &prov); + meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await + } + + /// Run the video enrichment pipeline for one resolved CID: filename-parser + /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each + /// plugin writes its own fields to meta-core. Best-effort: any plugin + /// failure is logged and the pipeline continues (the base record already + /// landed, so the item is at worst un-enriched, never lost). + pub async fn enrich_video( + &self, + cid: &str, + file_name: &str, + base_fields: BTreeMap, + ) { + let mut meta = base_fields; + + // 1. filename-parser → originalTitle / movieYear / videoType / season / episode. + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await { + Ok(()) => { + // Poll for the merged fields so tmdb gets the clean title. + if let Some(merged) = self + .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + { + meta = merged; + } else { + debug!( + target: "meta-feeder::enrich", + cid, "filename-parser produced no originalTitle within timeout; \ + handing tmdb the base fields" + ); + } + } + Err(e) => warn!( + target: "meta-feeder::enrich", + cid, error = %e, "filename-parser /process failed (non-fatal)" + ), + } + } + + // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video + + // a clean originalTitle, both now present on `meta`). + if let Some(tmdb_url) = self.cfg.tmdb_url.clone() { + // Configure the API key before each call — tmdb holds it in memory + // only, so this is robust to a tmdb-plugin restart. Cheap (local). + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /process failed (non-fatal)" + ); + } + } + + // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which + // tmdb writes async). Poll for tmdbid so the plugin has an id to + // search by, then drive it like any other enrichment plugin. + if self.cfg.opensubtitles_url.is_some() { + let merged = self + .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or(meta); + self.enrich_subtitles(cid, file_name, &merged).await; + } + } + + /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files + + /// an optional whole-torrent pack record. filename-parser runs on every + /// target; tmdb runs **once** on the primary video file and its show-level + /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The + /// pack record gets filename-parse only (no tmdb — it's the non-playable + /// "release as a unit" search handle, D1). Best-effort throughout. + pub async fn enrich_bundle(&self, videos: Vec, pack: Option) { + // Pack: filename-parse only, no tmdb. + if let Some(p) = pack { + self.fire_filename(&p).await; + } + + if videos.is_empty() { + return; + } + if videos.len() == 1 { + // Degenerate single-video bundle → the plain pipeline (filename → tmdb). + let v = &videos[0]; + self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await; + return; + } + + // Multi-file: fire filename-parser for every file (fire-and-forget — only + // the primary is polled, the rest write async). + for v in &videos { + self.fire_filename(v).await; + } + + // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen). + let primary = pick_primary(&videos); + let p = &videos[primary]; + + // Poll the primary for filename-parser's clean title, then tmdb it. + let merged = self + .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + .unwrap_or_else(|| p.base.clone()); + + let Some(tmdb_url) = self.cfg.tmdb_url.clone() else { + return; // no tmdb wired → filename-only enrichment, done. + }; + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed"); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed"); + return; + } + + // Read the resolved anchor back off the primary, then fan it to siblings. + let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await + else { + debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan"); + return; + }; + let anchor = extract_anchor(&anchor_rec); + if anchor.is_empty() { + return; + } + for (i, v) in videos.iter().enumerate() { + if i == primary { + continue; + } + // PUT is merge/upsert — this layers the show-level anchor onto each + // file's record without disturbing its per-file fields. + if let Err(e) = + meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await + { + warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); + } + } + + // Subtitles per video: each episode needs its own (its season/episode + // differs), but they share the fanned tmdbid. Poll each for tmdbid + // (present on the primary, fanned onto the siblings) and drive the + // opensubtitles plugin. Best-effort + quota-guarded inside the plugin. + if self.cfg.opensubtitles_url.is_some() { + for v in &videos { + let merged = self + .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or_else(|| v.base.clone()); + self.enrich_subtitles(&v.cid, &v.file_name, &merged).await; + } + } + } + + /// Fire filename-parser `/process` for one target (fire-and-forget; the + /// plugin merges its fields to meta-core async). + async fn fire_filename(&self, t: &EnrichTarget) { + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + if let Err(e) = self + .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base) + .await + { + warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed"); + } + } + } + + /// POST `/process` to a plugin. Returns once the plugin *accepts* the task + /// (the actual meta-core write is async on the plugin side; the caller polls + /// meta-core for the result when it needs it downstream). + async fn process( + &self, + plugin_url: &str, + plugin: &str, + cid: &str, + file_path: &str, + existing_meta: &BTreeMap, + ) -> anyhow::Result<()> { + let url = format!("{}/process", plugin_url.trim_end_matches('/')); + let body = ProcessRequest { + task_id: format!("{plugin}-{cid}-{}", now_nanos()), + cid, + file_path, + callback_url: &self.cfg.callback_url, + meta_core_url: &self.cfg.meta_core_url, + existing_meta, + }; + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("{plugin} POST {url} returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the tmdb plugin with the API key + language. + async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> { + let Some(token) = self.cfg.tmdb_token.clone() else { + anyhow::bail!("no TMDB token configured"); + }; + let url = format!("{}/configure", tmdb_url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { "apiKey": token, "language": self.cfg.tmdb_language } + }); + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("tmdb /configure returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the opensubtitles plugin with the API key, account + /// credentials, and wanted languages. + async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> { + let Some(key) = self.cfg.opensubtitles_api_key.clone() else { + anyhow::bail!("no OpenSubtitles API key configured"); + }; + let cfg_url = format!("{}/configure", url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { + "apiKey": key, + "username": self.cfg.opensubtitles_username, + "password": self.cfg.opensubtitles_password, + "languages": self.cfg.opensubtitles_languages, + } + }); + let resp = self.http.post(&cfg_url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("opensubtitles /configure returned {status}: {text}"); + } + Ok(()) + } + + /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or + /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta` + /// and writes the `subtitles//` key-set straight to meta-core). + /// Best-effort: any failure is logged and swallowed. + async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) { + let Some(url) = self.cfg.opensubtitles_url.clone() else { + return; + }; + if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") { + debug!( + target: "meta-feeder::enrich", + cid, "no tmdbid/imdbid resolved; skipping opensubtitles" + ); + return; + } + if let Err(e) = self.configure_opensubtitles(&url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /process failed (non-fatal)" + ); + } + } + + /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout` + /// elapses; returns the merged field map on success. + async fn poll_until_field( + &self, + cid: &str, + field: &str, + timeout: Duration, + ) -> Option> { + let deadline = SystemTime::now() + timeout; + loop { + match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await { + Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => { + return Some(fields); + } + Ok(_) => {} + Err(e) => debug!( + target: "meta-feeder::enrich", + cid, error = %e, "poll get_record failed" + ), + } + if SystemTime::now() >= deadline { + return None; + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } +} + +fn now_nanos() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// Pick the primary video file index: largest by `sizeByte`, ties broken toward +/// the first (lowest index). Falls back to 0 for an all-unsized bundle. +fn pick_primary(videos: &[EnrichTarget]) -> usize { + videos + .iter() + .enumerate() + .max_by_key(|(i, v)| { + let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0); + // larger size wins; on a tie the smaller index wins (Reverse). + (size, std::cmp::Reverse(*i)) + }) + .map(|(i, _)| i) + .unwrap_or(0) +} + +/// Project a resolved record down to the show-level anchor fields that are safe +/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]). +fn extract_anchor(fields: &BTreeMap) -> BTreeMap { + fields + .iter() + .filter(|(k, _)| { + ANCHOR_EXACT.contains(&k.as_str()) + || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p)) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_env_requires_meta_core_url() { + // Not set in the test env → None (legacy gateway-stores path). + // (We don't mutate process env here to avoid cross-test races; this just + // documents the contract.) + assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some()); + } + + fn tgt(cid: &str, size: u64) -> EnrichTarget { + EnrichTarget { + cid: cid.into(), + file_name: cid.into(), + base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]), + } + } + + #[test] + fn pick_primary_is_largest_then_first() { + let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)]; + // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins. + assert_eq!(pick_primary(&v), 1); + } + + #[test] + fn pick_primary_all_unsized_defaults_first() { + let v = vec![ + EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() }, + EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() }, + ]; + assert_eq!(pick_primary(&v), 0); + } + + #[test] + fn extract_anchor_keeps_show_level_drops_per_file() { + let rec = BTreeMap::from_iter([ + ("tmdbid".to_string(), "10378".to_string()), + ("imdbid".to_string(), "tt1254207".to_string()), + ("poster".to_string(), "bafkPOSTER".to_string()), + ("genres".to_string(), "Animation|Comedy".to_string()), + ("plot/eng".to_string(), "a plot".to_string()), + ("tags".to_string(), "tmdb-verified".to_string()), + // per-file — must NOT be fanned: + ("originalTitle".to_string(), "Big Buck Bunny".to_string()), + ("season".to_string(), "1".to_string()), + ("episode".to_string(), "3".to_string()), + ("fileName".to_string(), "ep3.mkv".to_string()), + ]); + let a = extract_anchor(&rec); + assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng")); + assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid")); + assert!(!a.contains_key("originalTitle"), "originalTitle is per-file"); + assert!(!a.contains_key("season") && !a.contains_key("episode")); + assert!(!a.contains_key("fileName")); + } +} diff --git a/crates/meta-feeder-sdk/src/filename_meta.rs b/crates/meta-feeder-sdk/src/filename_meta.rs new file mode 100644 index 0000000..8c93c11 --- /dev/null +++ b/crates/meta-feeder-sdk/src/filename_meta.rs @@ -0,0 +1,1309 @@ +//! Season / episode extraction from a torrent release title. +//! +//! This is a Rust port of `@metazla/filename-tools` +//! (`packages/filename-tool/src/lib/...`) — specifically the +//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` → +//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the +//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the +//! best season/episode heuristic we have (the meta-sort `torrent` plugin +//! already uses it), so rather than invent a second, weaker heuristic for +//! the gateway we mirror its pattern set and ordering verbatim. +//! +//! ## Differences from the TS original (deliberate, documented) +//! +//! 1. **No parent-folder fallback.** The TS extractor also reads +//! `Season N` out of the *parent directory* name. A torznab hit is a +//! single release title with no directory context, so that branch is +//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup +//! simply has nothing to read. +//! 2. **Rightmost-number fallback uses the captured digits, not the raw +//! regex match.** The TS `extractRightmostNumber` does +//! `parseFloat(fullMatch)` where the match can include a leading `.` +//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat` +//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the +//! bare number) instead, which is what the function *means* to return. +//! +//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so +//! a movie's release year can't be misread as an episode number by the +//! rightmost-number fallback. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; + +/// Structured season/episode extracted from a release title. Each field +/// is the stringified number exactly as the TS extractor would emit it +/// (`"" + n`): whole numbers render without a decimal, half-episode +/// specials keep the `.5` (e.g. `"13"`, `"13.5"`). +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SeasonEpisode { + pub season: Option, + pub episode: Option, + /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`). + /// Set together with [`episode_end`]; mutually exclusive with the + /// single-episode [`episode`] field. + pub episode_start: Option, + /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`). + pub episode_end: Option, + /// `season * 10000 + episode` disambiguator (matches the TS + /// `increment` field). Computed for completeness; the torznab caller + /// currently only persists `season`/`episode`. + pub increment: Option, + /// True when this release is a **pack/batch** rather than a single + /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season + /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with + /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack + /// is deliberately **not** given a single `episode`. + pub is_pack: bool, + /// True when [`season`] came from an explicit season-bearing token + /// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left + /// season-less (no "assume season 1" default), so a per-file reparse uses + /// this to know it should borrow the season from the pack/parent title (see + /// [`extract_season_episode_with_context`]). + pub season_explicit: bool, +} + +impl SeasonEpisode { + /// A pack carrying a concrete inclusive episode range. + pub fn has_episode_range(&self) -> bool { + self.episode_start.is_some() && self.episode_end.is_some() + } +} + +// --------------------------------------------------------------------------- +// Pattern fragments — copied 1:1 from +// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`. +// --------------------------------------------------------------------------- + +const SW: &str = r"(?:\.|\b)"; // a dot or word boundary +const S: &str = r"(?:\.|\b|\s)"; +const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored +const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5) +const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5) +const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5) +const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99 +const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5) + +/// Season-AND-episode patterns, longest/most-specific first. Each has +/// exactly two capture groups: group 1 = season, group 2 = episode. +/// Order matters — the first to match wins (mirrors the TS array order). +static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"), + format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"), + format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"), + format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"), + format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season+episode pattern compiles")) + .collect() +}); + +/// Episode-only patterns. Each has exactly one capture group: the episode. +static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"), + format!(r"(?i){SW}E{S}*{EP4MY}{END}"), + format!(r"(?i){SW}Ep{EP4MY}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("episode-only pattern compiles")) + .collect() +}); + +/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number. +static SOLO_EP: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles")); + +// --------------------------------------------------------------------------- +// Episode-range / pack detection. Anime + TV batches encode a *range* of +// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season +// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so +// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a +// batch collapses to one phantom episode. We detect ranges FIRST (before +// `find_season_and_episode`) and emit `episode_start`/`episode_end` + +// `is_pack`, so the caller can stamp `contentKind=pack` instead. +// +// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`: +// - ranges use a `-`/`~` separator with TWO numbers and `end > start`; +// - the bare form requires the start digit to be preceded by a space/`_`/`-` +// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read +// as a range start — that release falls through to the single-episode path; +// - 4-digit year-like / out-of-range operands are rejected. +// --------------------------------------------------------------------------- + +/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`. +static RANGE_BRACKET: LazyLock = LazyLock::new(|| { + Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles") +}); + +/// CJK episode range: `第29-38話` / `第29~38話`. +static RANGE_CJK: LazyLock = LazyLock::new(|| { + Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles") +}); + +/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces +/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a +/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight +/// separator is the disambiguator from the very common `Show 2 - 11` / +/// `Show - 11` episode-separator convention (spaced dash → single episode, not +/// a range); spaced ranges only come through the unambiguous bracketed form. +static RANGE_BARE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)") + .expect("bare range compiles") +}); + +/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single +/// season. Matched only as a fallback when no episode range is present. +static SEASON_RANGE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles") +}); + +/// `(start, end)` is a plausible inclusive episode range: ascending, and +/// neither operand is a 4-digit year / resolution-like number. +fn plausible_range(start: u32, end: u32) -> bool { + end > start && (1..1900).contains(&start) && end < 1900 +} + +/// Detect an episode range anywhere in the (precleaned) title. Returns the +/// first plausible `(start, end)` found, trying the unambiguous bracketed and +/// CJK forms before the bare form. +fn detect_episode_range(s: &str) -> Option<(u32, u32)> { + for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] { + for caps in re.captures_iter(s) { + let start = caps.get(1).and_then(|m| m.as_str().parse::().ok()); + let end = caps.get(2).and_then(|m| m.as_str().parse::().ok()); + if let (Some(a), Some(b)) = (start, end) { + if plausible_range(a, b) { + return Some((a, b)); + } + } + } + } + None +} + +/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix. +static SFV: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles")); + +// --------------------------------------------------------------------------- +// Keyword pre-clean — copied from +// `filename-tool/src/lib/config/KeywordsArray.ts`. +// Removed before season/episode detection so release tags like `x264`, +// `1080p`, `720` can't be mistaken for episode numbers by the fallback. +// --------------------------------------------------------------------------- + +/// Word-bounded keywords (resolution/codec/source/group tags). Order is +/// irrelevant once compiled individually, but we sort longest-first to +/// match the TS constructor (`keyword.length` desc) for identical +/// overlap behaviour. +const KEYWORDS: &[&str] = &[ + "VF", + "aXXo", + "STFR", + "SUBFORCED", + "BrRipx264", + "YIFY", + "TRUEFRENCH", + "VFF", + "ATVP", + "RenewalEX", + "BD", + "DD", + "8bits", + "8 bits", + "10bits", + "10 bits", + "part 1", + "part 2", + "part 3", + "part 4", + "part A", + "part B", + "part C", + "part D", + "AAC5", + "AAC5.1", + "1080i", + "1080p", + "2160p", + "480i", + "480p", + "4k", + "576i", + "576p", + "720", + "720i", + "720p", + "aac", + "aac4", + "ac3", + "amzn", + "apple tv+", + "avi", + "bbc", + "bd5", + "bdrip", + "blu-ray", + "bluray", + "brrip", + "cam", + "cw", + "dc", + "dcu", + "ddp5.1", + "director's cut", + "disney+", + "divx", + "divx5", + "dl", + "dsr", + "dsrip", + "dts", + "dual audio", + "dubbed", + "dvd", + "dvdivx", + "dvdr", + "dvdrip", + "dvdscr", + "dvdscreener", + "eng", + "eng sub", + "esp", + "fan edit", + "fhd", + "flv", + "fs", + "ger", + "german", + "h.264", + "h.265", + "h264", + "h265", + "hardcoded", + "hbo", + "hd", + "hddvd", + "hdr", + "hdrip", + "hdtv", + "hdtvrip", + "hevc", + "hq", + "hrhd", + "hrhdtv", + "hulu", + "imax", + "ita", + "jpn", + "ld", + "md", + "mkv", + "mp3", + "mp4", + "mpeg", + "mpg", + "mq", + "multi", + "multisubs", + "netflix", + "nf", + "nfofix", + "ntsc", + "ogg", + "ogm", + "ova", + "pal", + "pdtv", + "r3", + "r5", + "rerip", + "rsvcd", + "screener", + "sd", + "se", + "subbed", + "svcd", + "tc", + "telecine", + "telesync", + "ts", + "tv series", + "uhd", + "uhdtv", + "uhdv", + "v2", + "vcd", + "vostfr", + "web", + "web-dl", + "webcast", + "webrip", + "wmv", + "ws", + "www", + "x264", + "x265", + "xsvcd", + "xvid", + "xvidvd", + "xxx", + "bits", + "AnimeServ", +]; + +/// Substrings removed without word boundaries (the TS `substringArray`). +const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"]; + +/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first. +static KEYWORD_RES: LazyLock> = LazyLock::new(|| { + let mut kws: Vec<&str> = KEYWORDS.to_vec(); + kws.sort_by_key(|k| std::cmp::Reverse(k.len())); + kws.into_iter() + .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles")) + .collect() +}); + +/// Compiled substring regexes (no boundaries, case-insensitive), longest-first. +static SUBSTRING_RES: LazyLock> = LazyLock::new(|| { + let mut subs: Vec<&str> = SUBSTRINGS.to_vec(); + subs.sort_by_key(|s| std::cmp::Reverse(s.len())); + subs.into_iter() + .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles")) + .collect() +}); + +// --------------------------------------------------------------------------- +// Language tags — anime/torznab upstreams encode language as release-name +// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a +// structured field. `extract_languages` reads *every* tag off the raw title +// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`. +// The gateway writes each as a `languages/ = "true"` key-set member +// (METADATA_KEYS §9) that the preferred-language filter matches against. +// +// Patterns are ported from the battle-tested torrent-name parsers +// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead +// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`, +// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex` +// crate has no lookaround. Two defensive choices do most of the work: +// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They +// collide with ordinary words, names, and domain TLDs. We rely on the +// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead. +// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside +// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`. +// --------------------------------------------------------------------------- + +/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD +/// or path token can't be read as a language. Only `www.`-prefixed runs are +/// removed — a generic `host.tld` matcher would eat dotted release names like +/// `Spy.x.Family`. Runs on the raw (still-dotted) title. +static URL_STRIP: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles")); + +/// Guard phrases removed from the separator-normalized title before language +/// matching, because each contains a substring that would otherwise +/// false-positive a tag. None of these phrases is itself a language tag, so +/// removal is loss-free for detection — this is how we reproduce the upstream +/// parsers' lookbehind guards without lookaround: +/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese) +/// - `tel aviv` → guards `\btel\b` (Telugu) +/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker) +static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b") + .expect("lang guard re compiles") +}); + +/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match +/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js` +/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`), +/// because meta-share's "languages I read" preference is the source of truth +/// the meta-watch filter compares against — the enriched value must be in the +/// same space the chips use. (This deliberately diverges from `gutenberg.rs` / +/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.) +/// +/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for +/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release +/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul` +/// and `fre`: every pattern that matches contributes its code, so the result +/// is a set, not a first-wins pick. +static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| { + [ + // Multi / dual-audio → "mul". `dl` matches only after "web dl" has + // been stripped (see LANG_GUARD_STRIP). + ( + r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b", + "mul", + ), + // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…). + ( + r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b", + "fre", + ), + (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"), + (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"), + (r"\bitalian\b|\bita\b", "ita"), + (r"\bgerman\b|\bger\b|\bdeu\b", "ger"), + ( + r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b", + "spa", + ), + (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"), + (r"\bkorean\b|\bkor\b", "kor"), + ( + r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b", + "chi", + ), + (r"\brussian\b|\brus\b", "rus"), + (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"), + (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"), + (r"\bczech\b|\bcze\b|\bces\b", "cze"), + (r"\bslovak\b|\bslo\b|\bslk\b", "slo"), + (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"), + ( + r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b", + "rum", + ), + (r"\bgreek\b|\bgre\b|\bell\b", "gre"), + (r"\bukrainian\b|\bukr\b", "ukr"), + (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"), + (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"), + (r"\bnorwegian\b|\bnorsk\b", "nor"), + (r"\bdanish\b|\bdansk\b", "dan"), + (r"\bfinnish\b|\bsuomi\b", "fin"), + (r"\bturkish\b|\btur\b", "tur"), + (r"\barabic\b|\bara\b", "ara"), + (r"\bhindi\b|\bhin\b", "hin"), + (r"\btamil\b|\btam\b", "tam"), + (r"\btelugu\b|\btel\b", "tel"), + (r"\bmalayalam\b|\bmal\b", "mal"), + (r"\bvietnamese\b|\bvie\b", "vie"), + (r"\bthai\b|\btha\b", "tha"), + ] + .iter() + .map(|(p, code)| { + ( + Regex::new(&format!("(?i){p}")).expect("language tag re compiles"), + *code, + ) + }) + .collect() +}); + +/// Characters `cleanSpecialChars` replaces with a space (note: `.` and +/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07` +/// structure). +const SPECIAL_CHARS: &[char] = &[ + '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"', + '`', '~', +]; + +/// Extract season/episode from a release title. +/// +/// Faithful to `computeSeasonAndEpisodeFromFilename` → +/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath` +/// (minus the parent-folder branch — see the module doc). +pub fn extract_season_episode(title: &str) -> SeasonEpisode { + let cleaned = preclean(title); + // Range detection runs on the *raw* title (with `~` normalized to `-`), + // not the precleaned one: `preclean` replaces `~` with a space (it's a + // "special char"), which would erase a `26~37` range, and it never touches + // the `(`/`[` brackets the bracketed form anchors on. + let range_src = title.replace('~', "-"); + + // Pack detection runs first — an episode range (`(01-10)`) would otherwise + // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}` + // pattern. A pack is never given a single `episode`. + if let Some((start, end)) = detect_episode_range(&range_src) { + let season = find_pack_season(&cleaned); + return SeasonEpisode { + is_pack: true, + episode_start: Some(start.to_string()), + episode_end: Some(end.to_string()), + // An explicit season token (`S2 (01-10)`) buckets the pack; a bare + // range (`Frieren (01-28)`) leaves the season for the caller. + season: season.map(|s| s.to_string()), + season_explicit: season.is_some(), + ..Default::default() + }; + } + // Season range (`S01-S03`) — a whole-series pack, no single season/episode. + if SEASON_RANGE.is_match(&range_src) { + return SeasonEpisode { + is_pack: true, + ..Default::default() + }; + } + + let (sea, ep) = find_season_and_episode(&cleaned); + + let mut out = SeasonEpisode::default(); + // season/episode 0 are valid; only < 0 means "not found". + if sea >= 0.0 { + out.season = Some(fmt_num(sea)); + out.season_explicit = true; + } + if ep >= 0.0 { + out.episode = Some(fmt_num(ep)); + // An episode with no season is left season-LESS on purpose: a season is + // only asserted when explicitly hinted — an S-token in the title here, + // or (via extract_season_episode_with_context) the parent/pack title. + // Absolute-numbered releases ("Show - 28") thus carry an episode and no + // season, so the UI buckets them under "Episodes" rather than a + // fabricated "Season 1". `increment` encodes a season, so it is only + // emitted when the season is known. + if sea >= 0.0 { + out.increment = Some(fmt_num(sea * 10000.0 + ep)); + } + } else if sea >= 0.0 { + // Season present, no episode → a whole-season pack (`Season 3`, `S03`). + out.is_pack = true; + } + out +} + +/// Extract season/episode using a `/` style context — the +/// per-file path of an opened multi-file (pack) torrent. Parses the file's own +/// `filename` first; when the filename yields an episode but **no** season, the +/// season is recovered from the `parent` (the pack/torrent title). This +/// restores the parent-folder season fallback the TS +/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped +/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves +/// to S2E3 rather than the inherited pack-level number. +pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode { + let mut file_se = extract_season_episode(filename); + // Only borrow the parent's season for a concrete single episode whose own + // filename carries no *explicit* season token (so it is otherwise + // season-less). A per-file record is an episode, not a pack, so we don't + // propagate the parent's pack/range shape here. + if file_se.episode.is_some() && !file_se.season_explicit { + if let Some(s) = find_pack_season(&preclean(parent)) { + file_se.season = Some(s.to_string()); + file_se.season_explicit = true; + if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) { + file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep)); + } + } + } + file_se +} + +/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`, +/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to +/// recover a per-file season from the parent. Excludes the bare-number season +/// form so an episode-only title isn't mis-bucketed. +fn find_pack_season(s: &str) -> Option { + let padded = format!(" {s} "); + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let v = parse_group(&caps, 1); + if v >= 0.0 { + return Some(v as i64); + } + } + } + None +} + +/// Best-effort set of languages a release supports (audio **and** subtitles, +/// mixed), inferred from its title tags. +/// +/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`, +/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we +/// read them off the raw title here. Returns the **full set** of ISO 639-2/B +/// alpha-3 codes found (matching meta-share's UI vocabulary — see +/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is +/// empty when the title carries no recognizable tag — the caller turns that +/// into the `languages/und` sentinel. +pub fn extract_languages(title: &str) -> BTreeSet { + // 1. Strip `www.…` domain noise while the dots are still intact. + let stripped = URL_STRIP.replace_all(title, " "); + // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags + // anchor inside dotted/underscored/hyphenated release names — and so + // the guard phrases ("web dl", "shang chi") match regardless of which + // separator the original used. + let norm: String = stripped + .chars() + .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c }) + .collect(); + // 3. Remove guard phrases that would otherwise false-positive a tag. + let guarded = LANG_GUARD_STRIP.replace_all(&norm, " "); + // 4. Collect every matching code (a set — dedupes, deterministic order). + let mut out = BTreeSet::new(); + for (re, code) in LANGUAGE_TAGS.iter() { + if re.is_match(&guarded) { + out.insert((*code).to_string()); + } + } + out +} + +// --------------------------------------------------------------------------- +// Quality — the resolution *tier* of a release, normalized to a single label +// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured +// resolution, so — like season/episode and language — the release title is the +// only source. Mirrors the client-side `parseVersion()` heuristic in +// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's +// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to +// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height. +// --------------------------------------------------------------------------- + +/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH` +/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title +/// carries no recognizable resolution. +pub fn extract_quality(title: &str) -> Option { + static TIER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles") + }); + static WXH: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles")); + // Normalize `.`/`_` separators so `\b`-anchored tokens match inside + // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`). + let t = title.replace(['.', '_'], " "); + if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) { + let tok = m.as_str().to_lowercase(); + return Some( + match tok.as_str() { + "4k" => "2160p", + "1080i" => "1080p", + "720i" => "720p", + other => other, + } + .to_string(), + ); + } + if let Some(h) = WXH + .captures(&t) + .and_then(|c| c.get(1)) + .and_then(|m| m.as_str().parse::().ok()) + { + let tier = if h >= 2000 { + "2160p" + } else if h >= 1000 { + "1080p" + } else if h >= 700 { + "720p" + } else if h >= 500 { + "576p" + } else { + "480p" + }; + return Some(tier.to_string()); + } + None +} + +/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12 +/// structured search facet). Title-derived / best-effort — same provenance and +/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in +/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens +/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and +/// any stream-derived value. +pub fn extract_codec(title: &str) -> Option { + static CODEC: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b") + .expect("codec re compiles") + }); + // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored + // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`). + let t = title.replace(['.', '_'], " "); + let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase(); + let cleaned = tok.replace(' ', ""); + let norm = match cleaned.as_str() { + "x265" | "h265" | "hevc" => "hevc", + "x264" | "h264" | "avc" => "h264", + other => other, // av1, xvid, divx, vp9 + }; + Some(norm.to_string()) +} + +/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + +/// `cleanSpace`, in that order. +fn preclean(filename: &str) -> String { + let mut s = filename.to_string(); + for re in KEYWORD_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + // removeSFV — first occurrence only (TS uses a non-global replace). + s = SFV.replace(&s, "").into_owned(); + // cleanSpecialChars — replace each special char with a space. + s = s + .chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect(); + // cleanSpace — collapse runs of whitespace and trim. + s.split_whitespace().collect::>().join(" ") +} + +/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0` +/// for "not found". +fn find_season_and_episode(filename: &str) -> (f64, f64) { + // Pad with spaces so word-boundary fragments anchor at the ends + // exactly as the TS code does (`" " + filename + " "`). + let padded = format!(" {filename} "); + + for re in SEASON_EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + let episode = parse_group(&caps, 2); + return (season, episode); + } + } + for re in EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let episode = parse_group(&caps, 1); + return (-1.0, episode); + } + } + // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught + // here — *before* the bare rightmost-number fallback — so a whole-season + // pack reports `(season, no-episode)` instead of the fallback mis-reading + // its season digit as an episode and `extract_season_episode` then + // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03` + // bug). The TS pipeline relied on a parent-folder lookup for this case, + // which the gateway dropped (see module doc). + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + if season >= 0.0 { + return (season, -1.0); + } + } + } + // Last hope: the rightmost bare number. `if (number)` in the TS is + // falsy for 0, so a lone 0 doesn't count as an episode. + if let Some(n) = rightmost_number(&padded) { + if n != 0.0 { + return (-1.0, n); + } + } + (-1.0, -1.0) +} + +fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 { + caps.get(idx) + .and_then(|m| m.as_str().parse::().ok()) + .unwrap_or(-1.0) +} + +/// `extractRightmostNumber` — the last `soloEp` match's number. +fn rightmost_number(s: &str) -> Option { + SOLO_EP + .captures_iter(s) + .filter_map(|c| c.get(1)) + .last() + .and_then(|m| m.as_str().parse::().ok()) +} + +/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64 +/// `Display` already drops the trailing `.0`, so this is a thin wrapper +/// kept for intent. +fn fmt_num(n: f64) -> String { + format!("{n}") +} + +// --------------------------------------------------------------------------- +// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` + +// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`. +// The gateway previously used a separate hand-rolled `clean_torrent_title` +// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and +// season-word/episode-range stripping — so TMDB enrichment failed on many +// release-name shapes. Reuse the already-ported building blocks instead. +// --------------------------------------------------------------------------- + +/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` / +/// `S N` / trailing bare number and everything after it. +static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + format!(r"(?i){SW}{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-only pattern compiles")) + .collect() +}); + +/// Season-only patterns restricted to the **keyword'd** forms (`Season N`, +/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a +/// whole-season *pack* (no episode) before the bare rightmost-number fallback +/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}` +/// entry: that one matches any lone number and would turn an episode-only +/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because +/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`. +static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-word-only pattern compiles")) + .collect() +}); + +/// `cleanTags` — bracketed/parenthesised/braced groups. +static TAG_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles")); +/// `removeTagSeparator` — stray bracket chars. +static TAG_SEP_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles")); +/// `removeYear` — a standalone 19xx/20xx. +static YEAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles")); +/// `removeBeyondHyphen` — capture everything before the first ` - ` +/// (space-delimited) hyphen. +static BEYOND_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles")); +static TRAILING_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles")); + +fn clean_special_chars(s: &str) -> String { + s.chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect() +} + +fn remove_keywords(s: &str) -> String { + let mut out = s.to_string(); + for re in KEYWORD_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + out +} + +fn remove_beyond_hyphen(s: &str) -> String { + if let Some(c) = BEYOND_HYPHEN_RE.captures(s) { + return c + .get(1) + .map(|m| m.as_str().trim().to_string()) + .unwrap_or_default(); + } + TRAILING_HYPHEN_RE.replace(s, "").trim().to_string() +} + +fn remove_season_and_episode(s: &str) -> String { + for re in SEASON_EPISODE_PATTERNS + .iter() + .chain(EPISODE_PATTERNS.iter()) + .chain(SEASON_ONLY_PATTERNS.iter()) + { + if re.is_match(s) { + return re.replace(s, "").trim().to_string(); + } + } + // removeRightmostNumber fallback. + SOLO_EP.replace(s, "").trim().to_string() +} + +/// Produce a clean show/movie title from a raw release name, faithful to +/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB +/// search — NOT a hand-rolled cleaner. +pub fn clean_title(raw: &str) -> String { + let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`) + s = remove_keywords(&s); // 172 release/quality keywords + v1..v5 + s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags + s = remove_beyond_hyphen(&s); // drop everything after the first " - " + s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets + s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear + s = SFV.replace(&s, "").into_owned(); // removeSFV + s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number + s = s.replace('.', " "); // cleanDot + let s = s.trim().trim_start_matches('-').trim_end_matches('-'); + s.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn se(title: &str) -> (Option, Option) { + let r = extract_season_episode(title); + (r.season, r.episode) + } + + #[test] + fn strict_sxxeyy() { + assert_eq!( + se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"), + (Some("1".into()), Some("2".into())) + ); + } + + #[test] + fn dotted_release_name() { + assert_eq!( + se("The.Wire.S03E07.720p.BluRay.x264"), + (Some("3".into()), Some("7".into())) + ); + } + + #[test] + fn clean_title_collapses_release_variants() { + let ct = |s: &str| clean_title(s).to_lowercase(); + // Season-word, underscore/range, dotted+EnN, and the trailing-num + // Nyaa shape all collapse to the bare show title (the cases that + // were failing TMDB enrichment). + assert_eq!( + ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), + "spy x family" + ); + assert_eq!( + ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"), + "spy x family" + ); + assert_eq!( + ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"), + "spy x family" + ); + assert_eq!( + ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"), + "spy x family" + ); + } + + #[test] + fn concise_nxnn() { + assert_eq!( + se("Some Show 1x05 720p x264"), + (Some("1".into()), Some("5".into())) + ); + } + + #[test] + fn bracketed_concise() { + assert_eq!( + se("Some Show [2x10] HDTV"), + (Some("2".into()), Some("10".into())) + ); + } + + #[test] + fn verbose_season_episode() { + assert_eq!( + se("My Show Season 2 Episode 13"), + (Some("2".into()), Some("13".into())) + ); + } + + #[test] + fn anime_trailing_number_is_season_less() { + // No season marker → episode via rightmost-number fallback, and NO + // season is fabricated (absolute numbering). SFV id stripped, brackets + // ignored. The UI buckets a season-less episode under "Episodes". + assert_eq!( + se("[SubsPlease] Some Anime - 117 [FC412C51]"), + (None, Some("117".into())) + ); + } + + #[test] + fn half_episode_special_is_preserved() { + assert_eq!( + se("Some Anime S01E13.5"), + (Some("1".into()), Some("13.5".into())) + ); + } + + #[test] + fn increment_is_season_times_10000_plus_episode() { + let r = extract_season_episode("Show S02E05"); + assert_eq!(r.increment.as_deref(), Some("20005")); + } + + #[test] + fn four_digit_year_is_not_misread_as_episode() { + // The `soloEp` fallback caps at 3 digits, so a 4-digit release + // year can't be picked up — no episode here. + let r = extract_season_episode("The Matrix 1999 1080p BluRay x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn short_number_in_movie_title_is_grabbed_by_fallback() { + // A bare 1-3 digit number in a movie title DOES trip the + // rightmost-number fallback (here: "13"). This is exactly why the + // torznab caller gates season/episode on TV-ness — a movie record + // must not inherit this bogus episode. + let r = extract_season_episode("Apollo 13 1080p BluRay x264"); + assert_eq!(r.episode.as_deref(), Some("13")); + } + + #[test] + fn keyword_strip_prevents_codec_false_positive() { + // "x264" must be removed so "264" isn't read as the episode. + let r = extract_season_episode("Plain Movie Title x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn no_numbers_yields_nothing() { + let r = extract_season_episode("Just A Plain Title"); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + } + + #[test] + fn season_only_pack_is_not_misread_as_episode() { + // Whole-season packs (no episode) must report the season with NO + // episode — not `S01E0N` from the rightmost-number fallback. This is + // the original bug: `Spy x Family Season 3 (S03)` was parsed as + // season 1, episode 3, dumping the S3 pack into the Season-1 bucket. + assert_eq!( + se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"), + (Some("3".into()), None) + ); + assert_eq!( + se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"), + (Some("2".into()), None) + ); + // Bare `S03` pack (no `Season` word) via the S-prefixed pattern. + assert_eq!( + se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"), + (Some("3".into()), None) + ); + assert_eq!( + se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"), + (Some("3".into()), None) + ); + // French season word. + assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None)); + } + + #[test] + fn explicit_sxxeyy_still_wins_over_season_only() { + // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected + // by the new season-only branch. + assert_eq!( + se("Spy x Family Season 3 S03E13 1080p"), + (Some("3".into()), Some("13".into())) + ); + } + + #[test] + fn episode_range_is_a_pack_not_sxxeyy() { + // The headline bug: `(01-10)` was mis-read as S1 E10 by the + // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no + // single `episode`. + let r = extract_season_episode("Frieren (01-28) [Batch] 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("28")); + + // Bare, dash-delimited, leading-zero range. + let r = extract_season_episode("Some Anime 01-12 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("12")); + + // Tilde range, no leading zero. + let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("26")); + assert_eq!(r.episode_end.as_deref(), Some("37")); + } + + #[test] + fn explicit_season_buckets_a_pack() { + // `S2 (01-10)` → pack in season 2, episodes 1–10. + let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("10")); + } + + #[test] + fn cjk_episode_range_is_a_pack() { + let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("29")); + assert_eq!(r.episode_end.as_deref(), Some("38")); + } + + #[test] + fn season_range_is_a_pack_with_no_single_season() { + let r = extract_season_episode("Frieren S01-S03 Complete 1080p"); + assert!(r.is_pack); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start, None); + } + + #[test] + fn whole_season_release_is_a_pack() { + // A season-only release (`Season 3`, `S03`, no episode) is a pack too — + // the original "season-level release shows as one media card" bug. + let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("3")); + assert_eq!(r.episode, None); + } + + #[test] + fn single_episode_is_not_a_pack() { + // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all + // single episodes, never packs. + for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] { + assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack"); + } + let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("7")); + + // The `Show - ` anime convention (spaced dash) is a + // single episode, NOT a `2..11` range — the tight-separator rule keeps + // this out of pack detection. + let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("11")); + + // A year range in a movie title is not an episode range. + assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack); + } + + #[test] + fn per_file_context_recovers_season_from_parent() { + // Opened pack: the inner file carries the episode, the pack title the + // season. `Frieren - 03.mkv` alone is season-less; with the S2 pack + // parent it resolves to S2E3 (the parent is the "season hint"). + let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv"); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("3")); + assert!(!r.is_pack); + + // The file's own explicit season wins over the parent. + let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv"); + assert_eq!(r.season.as_deref(), Some("1")); + assert_eq!(r.episode.as_deref(), Some("5")); + } + + #[test] + fn absolute_numbered_anime_is_season_less() { + // No season keyword → the rightmost-number fallback still finds the + // episode, but NO season is fabricated. Season is only ever asserted + // from an explicit token (title or parent), never inferred — so an + // absolute-numbered release lands in the UI's "Episodes" bucket. + assert_eq!( + se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"), + (None, Some("37".into())) + ); + } + + fn langs(s: &str) -> BTreeSet { + extract_languages(s) + } + + #[test] + fn language_from_release_tags() { + // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi). + assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre")); + assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger")); + assert!(langs("Show ENG SUB 720p").contains("eng")); + assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn")); + assert!(langs("Film TRUEFRENCH 1080p").contains("fre")); + } + + #[test] + fn multi_is_mul_and_collects_every_language() { + // `languages` is the union of all supported audio+sub languages. + // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is + // ALSO collected (the set, not a first-wins pick). + assert_eq!( + langs("Show MULTI VOSTFR 1080p"), + BTreeSet::from(["mul".to_string(), "fre".to_string()]) + ); + assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul")); + assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul")); + // A bilingual release collects both specific codes. + assert_eq!( + langs("Movie.2021.GERMAN.ENGLISH.1080p"), + BTreeSet::from(["ger".to_string(), "eng".to_string()]) + ); + } + + #[test] + fn no_language_tag_yields_empty_set() { + // A plain quality/codec release name carries no language tag — the + // caller maps the empty set to the `languages/und` sentinel. + assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty()); + // Short codes embedded in words/names must not false-positive: "ita" + // inside "digital", "por" inside "portal", etc. + assert!(langs("VFX Reel 2024 1080p").is_empty()); + assert!(langs("Digital Capital Portal 1080p").is_empty()); + } + + #[test] + fn lookaround_guards_reproduced_in_code() { + // WEB-DL must not read as `dl`→mul (German dual-language marker). + assert!(!langs("Movie 2021 WEB-DL x264").contains("mul")); + // "Shang-Chi" must not read as `chi` (Chinese)… + assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi")); + // …but a real CHI tag still does. + assert!(langs("Movie.2021.CHI.1080p").contains("chi")); + // "Tel Aviv" must not read as `tel` (Telugu). + assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel")); + // www. domain noise (incl. its TLD) must not contribute a language. + assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty()); + } + + #[test] + fn quality_tiers_and_normalization() { + let q = |s: &str| extract_quality(s); + assert_eq!( + q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(), + Some("1080p") + ); + assert_eq!( + q("The.Wire.S03E07.720p.BluRay.x264").as_deref(), + Some("720p") + ); + // 4k / interlaced collapse to the canonical progressive tier. + assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p")); + assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p")); + assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p")); + // WxH fallback, mapped by height; dot/underscore separators handled. + assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p")); + assert_eq!(q("Show 1280x720").as_deref(), Some("720p")); + // No resolution token → None (never guess). + assert_eq!(q("Just A Plain Title"), None); + } + + #[test] + fn codec_normalization() { + let c = |s: &str| extract_codec(s); + // HEVC encoder spellings collapse to the codec name. + assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc")); + assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc")); + assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc")); + // AVC encoder spellings collapse to h264. + assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264")); + assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264")); + // Others kept as-is. + assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1")); + assert_eq!(c("Old Release XviD").as_deref(), Some("xvid")); + // No codec token → None (never guess). + assert_eq!(c("Just A Plain Title 1080p"), None); + } +} diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs new file mode 100644 index 0000000..0b87e2b --- /dev/null +++ b/crates/meta-feeder-sdk/src/hash.rs @@ -0,0 +1,889 @@ +//! Content-addressed identifiers used by feeder plugins' `compute_outcomes` +//! to hash upstream-fetched bytes. +//! +//! Two function families live here: +//! +//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] — +//! the size-prefix-plus-middle-1MB-sample identification primitive shared +//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`. +//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the +//! middle slice directly so plugins that can only fetch a partial range +//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file +//! just to throw most of it away. **Rust-only by construction.** +//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full +//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec +//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches +//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +//! --hash=sha2-256 `. +//! +//! Cross-implementation parity for `compute_midhash256` with the TypeScript +//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression +//! silently breaks dedupe across the platform. + +use bytes::Bytes; +use sha2::{Digest, Sha256}; + +/// Sample window in bytes (1 MiB). Files this size or smaller are hashed +/// in full; larger files are hashed over the middle 1 MiB slice. +const SAMPLE_SIZE: usize = 1024 * 1024; + +/// Per-block IPFS chunk size (256 KiB), matches kubo's +/// `--chunker=size-262144` default. +pub const IPFS_CHUNK_SIZE: usize = 256 * 1024; + +/// dag-pb fanout — children per internal node. Matches kubo's balanced +/// builder default. +pub const IPFS_FANOUT: usize = 174; + +/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every +/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by +/// their own CID. Pass `.blocks` to the gateway's blockstore so peers +/// can fetch the file by CID via bitswap. +#[derive(Debug, Clone)] +pub struct IpfsBlocks { + /// Canonical "bafy…" / "bafk…" root cid string. Identical to what + /// [`compute_ipfs_cid`] returns for the same input. + pub root: String, + /// Every block — root included — keyed by its own cid. For a + /// single-leaf file, this is a one-entry vec `[(root, payload)]` + /// (raw codec, payload is the file bytes). For multi-leaf files, + /// it is `leaves ++ internal_nodes` in build order; the **last** + /// entry is the root. + pub blocks: Vec<(String, Bytes)>, +} + +/// Compute a midhash256 CID matching the TypeScript implementation in +/// `meta-hash/src/lib/file-id/FastHash.ts`: +/// +/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the +/// whole file if it fits in 1 MiB). +/// 2. SHA-256 it. +/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field +/// and the multihash hash-code field, length 32, then multibase-base32-lower +/// with the `b` prefix. +/// +/// **Endianness for the size prefix is big-endian** to match TS's +/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every +/// import — keep the test pinned. +pub fn compute_midhash256(bytes: &[u8]) -> String { + let size = bytes.len(); + let sample: &[u8] = if size <= SAMPLE_SIZE { + bytes + } else { + let start = (size - SAMPLE_SIZE) / 2; + &bytes[start..start + SAMPLE_SIZE] + }; + compute_midhash256_from_sample(size as u64, sample) +} + +/// Same midhash256 primitive as [`compute_midhash256`], but takes the +/// sample bytes directly rather than slicing them out of a full file. +/// For upstreams that can only deliver a middle range (BitTorrent BEP 9, +/// sliced HTTP), this avoids forcing the caller to buffer the whole file +/// in memory just to throw most of it away. +/// +/// `total_size` is the **full file size in bytes** — used as the +/// size prefix, NOT `middle_sample.len()`. +pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String { + let size_be = total_size.to_be_bytes(); + let mut hasher = Sha256::new(); + hasher.update(size_be); + hasher.update(middle_sample); + let digest: [u8; 32] = hasher.finalize().into(); + + // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20]. + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(38); + wire.push(0x01); + wire.extend_from_slice(&[0x80, 0x20]); + wire.extend_from_slice(&[0x80, 0x20]); + wire.push(0x20); + wire.extend_from_slice(&digest); + + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1. +/// +/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so +/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11), +/// len=20, digest=`. +pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String { + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(4 + 20); + wire.push(0x01); // CIDv1 version + wire.push(0x55); // codec: raw + wire.push(0x11); // multihash: sha1 + wire.push(0x14); // multihash length: 20 + wire.extend_from_slice(infohash_20); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a single file inside a **BitTorrent v1** torrent +/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`. +pub const BTIH_V1_FILE_CODEC: u64 = 0x1001; + +/// Custom multicodec for a single file inside a **BitTorrent v2** torrent +/// (`btih-v2-file`). +pub const BTIH_V2_FILE_CODEC: u64 = 0x1002; + +/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent, +/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info` +/// dict) plus the zero-based file index in `info.files`. Single-file +/// torrents use index `0`. +pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String { + bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index) +} + +/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent +/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's +/// canonical traversal index in `file tree`. +pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String { + bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index) +} + +/// Shared encoder for the two torrent-file CID families. `infohash` is 20 +/// bytes for v1 / 32 for v2; `codec` selects the family. +fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String { + // digest = infohash ‖ varint(file_index) + let mut digest = Vec::with_capacity(infohash.len() + 2); + digest.extend_from_slice(infohash); + write_pb_varint(file_index, &mut digest); + + // CIDv1: [version][codec varint][multihash code varint][len varint][digest] + let mut wire = Vec::with_capacity(1 + 4 + digest.len()); + wire.push(0x01); + write_pb_varint(codec, &mut wire); + write_pb_varint(codec, &mut wire); + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a self-describing **Newznab release** locator +/// (`nzb-release`). MetaMesh-private, adjacent to the torrent-file codecs in the +/// `0x10xx` range. Unlike a content hash, the cid *embeds* the indexer host + +/// release id in an identity multihash, so the credentialed meta-share peer +/// decodes `{host,id}` straight from the cid and grabs the `.nzb` via `t=get` +/// only at playback — no side-table, no `.nzb` download at search time. (The +/// `0x1004` `sha256(host‖id)` opaque hash and the `0x1003` `nzb-posting` +/// variant were both removed; only this self-describing form is emitted.) +pub const NZB_RELEASE_CODEC: u64 = 0x1005; + +/// Digest byte ceiling for an `nzb-release` cid, matching meta-share's +/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. +const NZB_RELEASE_MAX_DIGEST: usize = 64; + +/// Encode a self-describing `nzb-release` CID from a Newznab indexer's host + +/// bare release id (the hex `` id). The multihash is *identity* (`0x00`) +/// and its digest embeds the locator as `varint(host_len) ‖ host ‖ id_bytes`, +/// where `id_bytes` is the hex-decoded id. meta-share's `decode_nzb_release_cid` +/// is the exact inverse. Host-namespaced: the same release on two indexers +/// yields different cids. +/// +/// Returns `None` when the id isn't hex or the digest would overflow the +/// multihash budget (an oversized host) — the caller drops that row. +pub fn compute_nzb_release_cid(host: &str, release_id: &str) -> Option { + let id_bytes = decode_hex_id(release_id)?; + let host_b = host.as_bytes(); + + // identity-multihash digest = varint(host_len) ‖ host ‖ id_bytes + let mut digest = Vec::with_capacity(2 + host_b.len() + id_bytes.len()); + write_pb_varint(host_b.len() as u64, &mut digest); + digest.extend_from_slice(host_b); + digest.extend_from_slice(&id_bytes); + if digest.len() > NZB_RELEASE_MAX_DIGEST { + return None; + } + + // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(NZB_RELEASE_CODEC, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + Some(format!("b{}", base32_lower_no_padding(&wire))) +} + +/// Hex-decode a Newznab release id. The guid parser only keeps hex runs, so the +/// id is all-hex by construction; odd-length ids are left-padded with a `0` +/// nibble. Returns `None` on any non-hex byte (defensive). +fn decode_hex_id(id: &str) -> Option> { + let padded; + let s: &str = if id.len() % 2 == 1 { + padded = format!("0{id}"); + &padded + } else { + id + }; + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len() / 2); + for pair in bytes.chunks(2) { + let hi = (pair[0] as char).to_digit(16)?; + let lo = (pair[1] as char).to_digit(16)?; + out.push(((hi << 4) | lo) as u8); + } + Some(out) +} + +/// Custom multicodec for a **card locator** — the identity of a *work* (a +/// series, a film) as published by a metadata bridge, rather than of any +/// bytes. MetaMesh-private, the next free slot after `0x1006` url. (`0x1004` +/// nzb-release-hash stays **retired — do not reuse**. `0x1003` was retired too, +/// but has since been **revived** for the self-scanned Usenet posting identity — +/// see [`NZB_POSTING_CODEC`] and `METADATA_KEYS.md` §2.) +/// +/// This is the first CID family in the platform where **no bytes exist +/// anywhere, ever** — `0x1005`/`0x1006` are locators too, but both eventually +/// resolve to bytes (NNTP, HTTP), and `0x1000` midhash256 is a digest over +/// real file bytes. A card is pure identity: the record *is* the payload, and +/// it lives in meta-core. Consequences, all already handled: +/// +/// * Bitswap seeding is gated on `HashKind::Sha2_256` (gateway invariant 6), +/// so a card is never offered to the swarm — nothing to seed. +/// * meta-share's byte path 404s for it, which is correct. +/// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for +/// `canonical_cid` — which is what makes the future "one card, two CIDs" +/// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe. +pub const CARD_LOCATOR_CODEC: u64 = 0x1007; + +/// Digest byte ceiling for a card locator, matching meta-share's +/// `MAX_MULTIHASH_SIZE` — its `CidGeneric<64>` rejects a longer multihash. +const CARD_LOCATOR_MAX_DIGEST: usize = 64; + +/// Encode a self-describing **card locator** CID from a metadata source and +/// that source's own id for the work (`("tmdb", "tv:95479")`, +/// `("mal", "52991")`). The multihash is *identity* (`0x00`) and its digest +/// embeds the locator as `varint(source_len) ‖ source ‖ id`, both UTF-8 — +/// exactly the shape [`compute_nzb_release_cid`] uses, minus the hex decode +/// (a card id is arbitrary text, not a hex release id). +/// +/// **The CID is a pure function of `(source, id)`**, which is the whole point: +/// any peer that knows a tmdb id can derive the card's address offline and +/// look it up locally, with no discovery round-trip. It also means the same +/// card discovered by two different peers converges on one meta-core record +/// for free, via the `cids/` reverse index. +/// +/// Source-namespaced: the same work on TMDB and on MyAnimeList yields +/// different cids. Merging those two into one record is a meta-core alias +/// operation driven by an explicit cross-source id mapping — never inferred +/// here (see `meta-gateway/docs/others/card-tier-search.md` §9). +/// +/// Returns `None` when `source` is empty or the digest would overflow the +/// multihash budget — the caller drops that card. +pub fn compute_card_cid(source: &str, id: &str) -> Option { + if source.is_empty() || id.is_empty() { + return None; + } + let source_b = source.as_bytes(); + let id_b = id.as_bytes(); + + // identity-multihash digest = varint(source_len) ‖ source ‖ id + let mut digest = Vec::with_capacity(2 + source_b.len() + id_b.len()); + write_pb_varint(source_b.len() as u64, &mut digest); + digest.extend_from_slice(source_b); + digest.extend_from_slice(id_b); + if digest.len() > CARD_LOCATOR_MAX_DIGEST { + return None; + } + + // CIDv1: [version=0x01][codec varint][mh code=0x00 identity][len varint][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(CARD_LOCATOR_CODEC, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + Some(format!("b{}", base32_lower_no_padding(&wire))) +} + +/// Custom multicodec for a **Usenet posting** identity (`nzb-posting`), +/// minted from the article Message-IDs of a release we scanned ourselves. +/// +/// Unlike [`NZB_RELEASE_CODEC`] (`0x1005`), this is **not** a locator: it +/// embeds no indexer host, so any peer with a plain NNTP provider can fetch the +/// articles — no `IndexerCred`, no `t=get` grab. That portability is the whole +/// point of running our own header scanner. +/// +/// It is also **not** fetchable by cid: the digest is over the Message-ID set, +/// not over any block's bytes, so bitswap can never satisfy a want for it (the +/// receiver derives a block's cid by hashing the block — see +/// `beetswap::incoming_stream`). The `.nzb` manifest travels separately, as an +/// ordinary sha2-256 IPFS cid in the record's `manifest` field. See +/// `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md` §5. +pub const NZB_POSTING_CODEC: u64 = 0x1003; + +/// Mint an `nzb-posting` cid (`0x1003`) from a release's article Message-IDs. +/// +/// # The normalisation rule IS the dedup contract +/// +/// Mirrored verbatim in meta-share (`nzb/manifest.rs`), which re-derives this +/// digest from the fetched manifest and rejects a mismatch. **Any change here +/// must land on both sides together** — a drift silently makes every posting +/// unplayable. The rule: +/// +/// 1. Trim each id, and strip a surrounding `<…>` pair if present (NZB +/// `` bodies carry the bare form, but be defensive — NNTP `BODY` +/// re-adds the brackets). +/// 2. Drop empties. +/// 3. Deduplicate, then **sort** lexicographically by bytes. +/// 4. Join with `\n` and SHA-256 the result. +/// +/// Case is **preserved**: a Message-ID's local part is case-sensitive per RFC +/// 5322, so lowercasing could collide two genuinely distinct articles. +/// +/// Sorting is what makes the cid survive segment/file reordering between two +/// NZBs of the same posting — the reason we hash the *id set* and never the raw +/// `.nzb` bytes (the retired `0x1004` did the latter and broke dedup; see +/// meta-share `nzb/mod.rs`). +/// +/// **v1 scope:** every segment counts, par2 volumes included. Two NZBs for one +/// posting that disagree about whether to bundle par2 therefore mint different +/// cids. Acceptable while we are the only producer of these manifests; if a +/// second producer ever appears, the fix is to filter par2 subjects here *and* +/// in meta-share's mirror — do not do it on one side only. +pub fn compute_nzb_posting_cid>(message_ids: &[S]) -> String { + let mut ids: Vec<&str> = message_ids + .iter() + .map(|s| { + let t = s.as_ref().trim(); + t.strip_prefix('<') + .and_then(|r| r.strip_suffix('>')) + .unwrap_or(t) + }) + .filter(|s| !s.is_empty()) + .collect(); + ids.sort_unstable(); + ids.dedup(); + + let mut hasher = Sha256::new(); + for (i, id) in ids.iter().enumerate() { + if i > 0 { + hasher.update(b"\n"); + } + hasher.update(id.as_bytes()); + } + let digest: [u8; 32] = hasher.finalize().into(); + + // CIDv1: [version=0x01][codec varint][mh code=0x12 sha2-256][len=32][digest] + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); + write_pb_varint(NZB_POSTING_CODEC, &mut wire); + wire.push(0x12); // multihash code: sha2-256 + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's +/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +/// --hash=sha2-256 `. +pub fn compute_ipfs_cid(bytes: &[u8]) -> String { + compute_ipfs_blocks(bytes).root +} + +/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every +/// block produced along the way (leaf chunks + internal dag-pb nodes) +/// so callers can populate a bitswap blockstore in one pass. +pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks { + // Leaves: chunk bytes into raw-codec blocks. Empty input still gets + // one (empty) leaf so the cid is well-defined. + let mut blocks: Vec<(String, Bytes)> = Vec::new(); + let leaves: Vec<(Vec, u64)> = if bytes.is_empty() { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b"")); + blocks.push((cid_string(&cid_wire), Bytes::new())); + vec![(cid_wire, 0)] + } else { + bytes + .chunks(IPFS_CHUNK_SIZE) + .map(|chunk| { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk)); + blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk))); + (cid_wire, chunk.len() as u64) + }) + .collect() + }; + + // Single-leaf file: return the leaf cid as the file cid. + if leaves.len() == 1 { + let root = cid_string(&leaves[0].0); + return IpfsBlocks { root, blocks }; + } + + // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes. + let mut level: Vec<(Vec, u64, u64)> = leaves + .into_iter() + .map(|(cid, size)| (cid, size, size)) + .collect(); + + while level.len() > 1 { + let mut next: Vec<(Vec, u64, u64)> = + Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT)); + for batch in level.chunks(IPFS_FANOUT) { + let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect(); + let filesize: u64 = blocksizes.iter().sum(); + let unixfs_data = encode_unixfs_file(filesize, &blocksizes); + let node_bytes = encode_dagpb_node(batch, &unixfs_data); + let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes)); + blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone()))); + let own_tsize: u64 = + batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64; + next.push((node_cid, filesize, own_tsize)); + } + level = next; + } + let root = cid_string(&level[0].0); + IpfsBlocks { root, blocks } +} + +fn cid_string(wire: &[u8]) -> String { + format!("b{}", base32_lower_no_padding(wire)) +} + +/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code +/// varint=0x12 sha2-256][len=0x20][digest...]`. +fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec { + debug_assert!( + codec < 0x80, + "codec varint must fit in one byte for this helper" + ); + let mut wire = Vec::with_capacity(36); + wire.push(0x01); // CIDv1 + wire.push(codec); + wire.push(0x12); // multihash code: sha2-256 + wire.push(0x20); // digest length: 32 + wire.extend_from_slice(digest); + wire +} + +fn sha2_256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// LEB128 unsigned varint encoding into `out`. +fn write_pb_varint(value: u64, out: &mut Vec) { + let mut v = value; + while v >= 0x80 { + out.push((v as u8) | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) { + write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint) + write_pb_varint(value, out); +} + +fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) { + write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited + write_pb_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +/// UnixFS protobuf payload for a File node: `Type=File, filesize, +/// blocksizes[]`. +fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec { + let mut out = Vec::new(); + write_pb_varint_field(1, 2, &mut out); // Type = File + write_pb_varint_field(3, filesize, &mut out); // filesize + for &bs in blocksizes { + write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated) + } + out +} + +/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order +/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second. +fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec { + let mut out = Vec::new(); + for (cid_wire, _filesize, tsize) in children { + let mut link_bytes = Vec::new(); + write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash + write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize + write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first + } + write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last + out +} + +/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the +/// multibase `b` prefix. +pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut out = String::with_capacity(input.len().div_ceil(5) * 8); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &b in input { + buffer = (buffer << 8) | b as u64; + bits += 8; + while bits >= 5 { + bits -= 5; + let idx = ((buffer >> bits) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + } + if bits > 0 { + let idx = ((buffer << (5 - bits)) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn midhash256_matches_ts_fixtures() { + assert_eq!( + compute_midhash256(b"hello world"), + "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4" + ); + assert_eq!( + compute_midhash256(b""), + "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y" + ); + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_midhash256(&mb_zeros), + "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4" + ); + } + + #[test] + fn midhash256_from_sample_matches_full_bytes() { + let small: &[u8] = b"hello world"; + assert_eq!( + compute_midhash256_from_sample(small.len() as u64, small), + compute_midhash256(small), + ); + + assert_eq!( + compute_midhash256_from_sample(0, b""), + compute_midhash256(b""), + ); + + let one_mb = vec![0u8; SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(one_mb.len() as u64, &one_mb), + compute_midhash256(&one_mb), + ); + + let two_mb = vec![0u8; 2 * SAMPLE_SIZE]; + let start = (two_mb.len() - SAMPLE_SIZE) / 2; + let middle = &two_mb[start..start + SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(two_mb.len() as u64, middle), + compute_midhash256(&two_mb), + ); + } + + #[test] + fn midhash256_from_sample_partial_sample_stable() { + let sample = vec![0u8; SAMPLE_SIZE]; + let total_5gb: u64 = 5 * 1024 * 1024 * 1024; + let total_10gb: u64 = 10 * 1024 * 1024 * 1024; + + let h1 = compute_midhash256_from_sample(total_5gb, &sample); + let h2 = compute_midhash256_from_sample(total_5gb, &sample); + assert_eq!(h1, h2, "same (size, sample) must hash identically"); + + let h3 = compute_midhash256_from_sample(total_10gb, &sample); + assert_ne!( + h1, h3, + "size prefix must contribute to cid — same sample, different size" + ); + } + + #[test] + fn bt_v1_file_cid_wire_layout() { + let infohash: [u8; 20] = [ + 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c, + 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06, + ]; + + let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want.extend_from_slice(&infohash); + want.push(0x04); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 4), + format!("b{}", base32_lower_no_padding(&want)) + ); + + let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want0.extend_from_slice(&infohash); + want0.push(0x00); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want0)) + ); + + let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16]; + want200.extend_from_slice(&infohash); + want200.extend_from_slice(&[0xC8, 0x01]); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 200), + format!("b{}", base32_lower_no_padding(&want200)) + ); + } + + #[test] + fn bt_v2_file_cid_wire_layout() { + let infohash: [u8; 32] = [0xAB; 32]; + let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21]; + want.extend_from_slice(&infohash); + want.push(0x00); + assert_eq!( + compute_bt_v2_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want)) + ); + } + + #[test] + fn bt_file_cid_properties() { + let ih1 = [0x11u8; 20]; + let ih2 = [0x22u8; 32]; + + assert_eq!( + compute_bt_v1_file_cid(&ih1, 3), + compute_bt_v1_file_cid(&ih1, 3) + ); + assert_ne!( + compute_bt_v1_file_cid(&ih1, 0), + compute_bt_v1_file_cid(&ih1, 1) + ); + assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b')); + assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b')); + assert_ne!( + compute_bt_v1_file_cid(&[0xAB; 20], 0), + compute_bt_v2_file_cid(&[0xAB; 32], 0) + ); + } + + #[test] + fn base32_known_vectors() { + assert_eq!(base32_lower_no_padding(b""), ""); + assert_eq!(base32_lower_no_padding(b"f"), "my"); + assert_eq!(base32_lower_no_padding(b"fo"), "mzxq"); + assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6"); + assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq"); + assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi"); + } + + #[test] + fn ipfs_cid_single_leaf_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b"hello world"), + "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" + ); + } + + #[test] + fn ipfs_cid_empty_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b""), + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + } + + #[test] + fn ipfs_cid_multi_leaf_zeros_stable() { + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_ipfs_cid(&mb_zeros), + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + } + + #[test] + fn ipfs_blocks_single_leaf() { + let payload = b"hello world"; + let out = compute_ipfs_blocks(payload); + assert_eq!(out.root, compute_ipfs_cid(payload)); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].0, out.root); + assert_eq!(out.blocks[0].1.as_ref(), payload); + } + + #[test] + fn ipfs_blocks_multi_leaf() { + let mb_zeros = vec![0u8; 1024 * 1024]; + let out = compute_ipfs_blocks(&mb_zeros); + assert_eq!( + out.root, + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + assert_eq!(out.blocks.len(), 5); + assert_eq!(out.blocks.last().unwrap().0, out.root); + for (_cid, block) in &out.blocks[..4] { + assert_eq!(block.len(), 256 * 1024); + assert!(block.iter().all(|&b| b == 0)); + } + } + + #[test] + fn ipfs_blocks_empty() { + let out = compute_ipfs_blocks(b""); + assert_eq!( + out.root, + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].1.len(), 0); + assert_eq!(out.blocks[0].0, out.root); + } + + // ---- card locator (0x1007) -------------------------------------------- + + /// Inverse of [`base32_lower_no_padding`], test-only. Lets the card tests + /// assert the actual CIDv1 bytes (codec, multihash code, digest) rather + /// than just round-tripping the encoder against itself. + fn base32_lower_decode(s: &str) -> Vec { + const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut out = Vec::new(); + let (mut acc, mut bits) = (0u32, 0u32); + for c in s.bytes() { + let v = ALPHABET.iter().position(|&a| a == c).expect("base32 char") as u32; + acc = (acc << 5) | v; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + out + } + + /// The property the whole card design leans on: the CID is a pure function + /// of `(source, id)`, so any peer can derive a card's address offline. If + /// this literal ever changes, every already-published card silently forks + /// into a second identity — treat a failure here as a wire break. + #[test] + fn card_cid_is_deterministic() { + let cid = compute_card_cid("tmdb", "tv:95479").expect("card cid"); + assert_eq!(cid, compute_card_cid("tmdb", "tv:95479").unwrap()); + assert_eq!(cid, "bagdsaaanar2g2zdcor3duojvgq3ts"); + } + + /// Source- and kind-namespaced: the same numeric id on a different bridge, + /// or the same id under a different media kind, must not collide. + #[test] + fn card_cid_is_source_and_kind_namespaced() { + let tmdb = compute_card_cid("tmdb", "tv:95479").unwrap(); + let mal = compute_card_cid("mal", "95479").unwrap(); + let movie = compute_card_cid("tmdb", "movie:95479").unwrap(); + assert_ne!(tmdb, mal); + assert_ne!(tmdb, movie); + } + + /// Pins the on-the-wire CIDv1 layout: v1, codec `0x1007` (varint + /// `[0x87,0x20]`), **identity** multihash `0x00` (this family has no + /// digest — the locator itself is the payload), then + /// `varint(source_len) ‖ source ‖ id`. + #[test] + fn card_cid_wire_shape() { + let cid = compute_card_cid("tmdb", "tv:95479").unwrap(); + assert!(cid.starts_with('b'), "multibase base32-lower prefix"); + let wire = base32_lower_decode(&cid[1..]); + + assert_eq!(wire[0], 0x01, "CIDv1"); + assert_eq!(&wire[1..3], &[0x87, 0x20], "codec varint for 0x1007"); + assert_eq!(wire[3], 0x00, "identity multihash — no digest"); + + let digest_len = wire[4] as usize; + let digest = &wire[5..5 + digest_len]; + assert_eq!(digest[0], 4, "varint(len(\"tmdb\"))"); + assert_eq!(&digest[1..5], b"tmdb"); + assert_eq!(&digest[5..], b"tv:95479"); + } + + /// Empty inputs are rejected rather than encoded as an ambiguous locator, + /// and an id past the 64-byte multihash budget (meta-share's + /// `CidGeneric<64>`) returns `None` so the caller drops the card instead of + /// emitting a cid that peer would refuse to parse. + #[test] + fn card_cid_rejects_empty_and_oversized() { + assert!(compute_card_cid("", "tv:1").is_none()); + assert!(compute_card_cid("tmdb", "").is_none()); + let huge = "x".repeat(CARD_LOCATOR_MAX_DIGEST); + assert!(compute_card_cid("tmdb", &huge).is_none()); + // Exactly at the ceiling still encodes. + let fits = "x".repeat(CARD_LOCATOR_MAX_DIGEST - 5); // varint(4) + "tmdb" + assert!(compute_card_cid("tmdb", &fits).is_some()); + } + + // ---- nzb-posting (0x1003) — the dedup contract ----------------------- + // + // These pin the normalisation rule documented on `compute_nzb_posting_cid`. + // meta-share re-derives the same digest from the fetched manifest and + // rejects a mismatch, so a change that breaks one of these silently makes + // every self-scanned posting unplayable. Mirror any edit on both sides. + + /// Segment/file **order must not matter** — that is the whole reason we + /// hash a sorted id set instead of the raw `.nzb` bytes (the retired + /// `0x1004` did the latter and broke dedup across reposts). + #[test] + fn nzb_posting_cid_is_order_independent() { + let a = ["c@x.com", "a@x.com", "b@x.com"]; + let b = ["a@x.com", "b@x.com", "c@x.com"]; + let c = ["b@x.com", "c@x.com", "a@x.com"]; + assert_eq!(compute_nzb_posting_cid(&a), compute_nzb_posting_cid(&b)); + assert_eq!(compute_nzb_posting_cid(&a), compute_nzb_posting_cid(&c)); + } + + /// Angle brackets, surrounding whitespace, empty entries and duplicates are + /// all normalised away. NZB `` bodies carry the bare form but NNTP + /// `BODY` re-adds the brackets, so both spellings must converge. + #[test] + fn nzb_posting_cid_normalises_brackets_blanks_and_dupes() { + let bare = ["a@x.com", "b@x.com"]; + let noisy = [" ", "b@x.com", "", " ", "a@x.com"]; + assert_eq!( + compute_nzb_posting_cid(&bare), + compute_nzb_posting_cid(&noisy) + ); + } + + /// Case is preserved: a Message-ID's local part is case-sensitive per RFC + /// 5322, so lowercasing could collide two genuinely distinct articles. + #[test] + fn nzb_posting_cid_is_case_sensitive() { + assert_ne!( + compute_nzb_posting_cid(&["Abc@x.com"]), + compute_nzb_posting_cid(&["abc@x.com"]) + ); + } + + /// A different article set is a different posting. + #[test] + fn nzb_posting_cid_differs_on_different_ids() { + assert_ne!( + compute_nzb_posting_cid(&["a@x.com", "b@x.com"]), + compute_nzb_posting_cid(&["a@x.com", "c@x.com"]) + ); + } + + /// Wire shape: CIDv1, codec `0x1003` (varint `[0x83, 0x20]`), multihash + /// sha2-256 (`0x12`), 32-byte digest. Pinned because meta-share matches on + /// the **codec** slot — a drift here makes every posting undecodable there. + #[test] + fn nzb_posting_cid_wire_shape() { + let cid = compute_nzb_posting_cid(&["a@x.com"]); + assert!(cid.starts_with('b'), "multibase base32-lower prefix"); + + // Round-trip the base32 body back to bytes to assert the header. + let decoded = base32_lower_decode(&cid[1..]); + assert_eq!(decoded[0], 0x01, "CIDv1"); + assert_eq!(&decoded[1..3], &[0x83, 0x20], "varint(0x1003)"); + assert_eq!(decoded[3], 0x12, "multihash sha2-256"); + assert_eq!(decoded[4], 0x20, "digest length 32"); + assert_eq!(decoded.len(), 5 + 32); + + // The digest is sha256 over the newline-joined sorted set. + let expect: [u8; 32] = Sha256::digest(b"a@x.com").into(); + assert_eq!(&decoded[5..], &expect); + } +} diff --git a/crates/meta-feeder-sdk/src/lang.rs b/crates/meta-feeder-sdk/src/lang.rs new file mode 100644 index 0000000..3d306c0 --- /dev/null +++ b/crates/meta-feeder-sdk/src/lang.rs @@ -0,0 +1,81 @@ +//! Language-code normalization — ISO 639 alias folding, **not** filename +//! parsing. +//! +//! This lives apart from [`crate::filename_meta`] on purpose: it doesn't read a +//! release title, it canonicalizes a *language code* (an indexer's declared +//! language, a query's `languages:` filter, a title-derived stamp) onto the +//! single ISO 639-2/B alpha-3 vocabulary the rest of the stack compares against. +//! It is the one piece of the old `filename_meta` module that has no equivalent +//! in the `metamesh-plugin-filename-parser` plugin — because it isn't filename +//! parsing — so it stays in Rust while the title-parsing algorithm itself is +//! owned by the plugin. Callers in the **request hot path** (e.g. the torznab +//! indexer-language fan-out skip) need it synchronously, with no network hop. + +/// Normalize a language code to the ISO 639-2/B alpha-3 vocabulary the stack +/// emits (`fre`/`ger`/`chi`, **not** the 639-2/T `fra`/`deu`/`zho` nor the +/// 639-1 two-letter forms), lowercased. This is the space meta-share's +/// "languages I read" preference uses and that the title-derived language stamps +/// write, so an operator who declares an indexer's language as `fra`/`de`/`zh` +/// still lines up with a `fre`/`ger`/`chi` query filter. Unknown / +/// already-canonical codes pass through lowercased+trimmed unchanged. +pub fn normalize_lang_code(code: &str) -> String { + let c = code.trim().to_ascii_lowercase(); + let mapped = match c.as_str() { + // 639-2/T and 639-1 → 639-2/B for the codes that actually differ. + "fra" | "fr" => "fre", + "deu" | "de" => "ger", + "zho" | "zh" => "chi", + "nld" | "nl" => "dut", + "ell" | "el" => "gre", + "ron" | "ro" => "rum", + "slk" | "sk" => "slo", + "ces" | "cs" => "cze", + // 639-1 → 639-2 for the rest of the language-tag output space (these + // share the same /B and /T alpha-3, so only the two-letter form maps). + "en" => "eng", + "ja" => "jpn", + "it" => "ita", + "es" => "spa", + "pt" => "por", + "ko" => "kor", + "ru" => "rus", + "pl" => "pol", + "hu" => "hun", + "uk" => "ukr", + "bg" => "bul", + "sv" => "swe", + "no" => "nor", + "da" => "dan", + "fi" => "fin", + "tr" => "tur", + "ar" => "ara", + "hi" => "hin", + "ta" => "tam", + "te" => "tel", + "ml" => "mal", + "vi" => "vie", + "th" => "tha", + _ => return c, + }; + mapped.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_lang_maps_to_639_2b_vocabulary() { + // 639-2/T and 639-1 spellings collapse onto the /B forms the stack uses. + assert_eq!(normalize_lang_code("fra"), "fre"); + assert_eq!(normalize_lang_code("FR"), "fre"); + assert_eq!(normalize_lang_code("deu"), "ger"); + assert_eq!(normalize_lang_code("zh"), "chi"); + assert_eq!(normalize_lang_code("nld"), "dut"); + // Already-canonical / unknown codes pass through trimmed + lowercased. + assert_eq!(normalize_lang_code(" ENG "), "eng"); + assert_eq!(normalize_lang_code("fre"), "fre"); + assert_eq!(normalize_lang_code("mul"), "mul"); + assert_eq!(normalize_lang_code("xyz"), "xyz"); + } +} diff --git a/crates/meta-feeder-sdk/src/lib.rs b/crates/meta-feeder-sdk/src/lib.rs new file mode 100644 index 0000000..dd62603 --- /dev/null +++ b/crates/meta-feeder-sdk/src/lib.rs @@ -0,0 +1,49 @@ +//! `meta-feeder-sdk` — the source-agnostic foundation shared by the gateway +//! core and every feeder sidecar binary. +//! +//! A *feeder* implements [`FeederPlugin`] (find records, fetch bytes) for one +//! or more upstreams, and [`serve_feeders`] exposes them over the feeder HTTP +//! contract. The gateway core consumes that contract via its +//! `RemoteFeederPlugin` and keeps the libp2p wire, the bitswap blockstore, the +//! hashing-into-the-blockstore, and the meta-core store-back to itself. +//! +//! Deliberately libp2p-free and blockstore-free — see this crate's `Cargo.toml`. + +// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it +// directly rather than boxing on every embedded-DB call. Boxing each would be +// pure noise for a single-process store, so silence the lint crate-wide. +#![allow(clippy::result_large_err)] + +/// pcs-convention default meta-core root a self-publishing feeder targets when +/// neither its dashboard config nor the `META_CORE_URL` env seed sets one. Keeps +/// the modular storage seam out of compose: a standard install just works; a +/// non-standard meta-core is set in the feeder's config UI. Matches the +/// `metacore-app` service the AppStore feeder apps ship on the shared network. +pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000"; + +pub mod cache; +pub mod common; +pub mod config; +pub mod enrich; +pub mod filename_meta; +pub mod hash; +pub mod lang; +pub mod meta_core; +pub mod plugin; +pub mod query; +pub mod query_eval; +pub mod serve; +pub mod types; + +pub use config::{ConfigField, ConfigSchema, FieldKind}; +pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher}; +pub use meta_core::FeederStore; +pub use plugin::{ + upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, +}; +pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; +pub use serve::{ + configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, + HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, +}; +pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; diff --git a/crates/meta-feeder-sdk/src/meta_core.rs b/crates/meta-feeder-sdk/src/meta_core.rs new file mode 100644 index 0000000..93cfe52 --- /dev/null +++ b/crates/meta-feeder-sdk/src/meta_core.rs @@ -0,0 +1,293 @@ +//! Lightweight meta-core write/read client for feeders that **self-publish**. +//! +//! Post-"smart feeder / dumb gateway" split: a feeder given a `META_CORE_URL` +//! writes its own base record into meta-core instead of returning it for the +//! gateway dispatcher to store. This keeps the gateway a pure aggregator + +//! request router; the feeder owns the full record lifecycle for its upstream. +//! +//! The write shape is **byte-for-byte the same** as the gateway's +//! `meta_core::build_metadata_body` (`crates/meta-gateway/src/meta_core.rs`): +//! `cid_*` fields collapse to the `cids/ = "true"` key-set, and a +//! `provenance` JSON blob is stamped. Keeping them identical means a record +//! written by the feeder is indistinguishable from one the dispatcher wrote — +//! the meta-share ingester, meta-watch, and ranking all behave the same. +//! +//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core +//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the +//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}` +//! merges accumulate on the same `/file/{cid}` hash. + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use tracing::warn; + +use crate::plugin::{HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, Hash}; + +/// Provenance recorded on every gateway-acquired record. Mirrors the gateway +/// dispatcher's shape so feeder-written and dispatcher-written records are +/// interchangeable. +#[derive(Clone, Debug)] +pub struct Provenance { + pub upstream: String, + pub record_id: String, + /// Stable per-deployment id (the gateway peer this feeder backs, from + /// `META_GATEWAY_PEER_ID`, else the feeder hostname). Analytics group by it. + pub gateway_peer: String, + pub acquired_at: u64, +} + +impl Provenance { + /// Build a provenance stamp for `upstream`/`record_id`, stamping "now". + pub fn now(upstream: &str, record_id: &str, gateway_peer: &str) -> Self { + Provenance { + upstream: upstream.to_string(), + record_id: record_id.to_string(), + gateway_peer: gateway_peer.to_string(), + acquired_at: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + } + } +} + +/// JSON-stringified provenance value. Same key set + `"source":"gateway"` as the +/// gateway dispatcher (`crates/meta-gateway/src/meta_core.rs::provenance_to_json`). +pub fn provenance_to_json(p: &Provenance) -> String { + serde_json::to_string(&serde_json::json!({ + "source": "gateway", + "upstream": p.upstream, + "recordId": p.record_id, + "gatewayPeer": p.gateway_peer, + "acquiredAt": p.acquired_at, + })) + .expect("static JSON shape — serialisation is infallible") +} + +/// Build the flat meta-core PUT body from a [`DiscoveryRecord`] + provenance. +/// +/// Identical transform to the gateway dispatcher: any `cid_*` field becomes a +/// `cids/ = "true"` key-set member (the CID is self-describing — the +/// algorithm is the multicodec; see METADATA_KEYS.md §2/§14.13), everything else +/// passes through, and `provenance` is stamped last (overriding any plugin-set +/// value). +pub fn build_metadata_body( + record: &DiscoveryRecord, + provenance: &Provenance, +) -> BTreeMap { + let mut body: BTreeMap = BTreeMap::new(); + for (k, v) in &record.fields { + if k.starts_with("cid_") && !v.is_empty() { + body.insert(format!("cids/{v}"), "true".to_string()); + } else { + body.insert(k.clone(), v.clone()); + } + } + body.insert("provenance".to_string(), provenance_to_json(provenance)); + body +} + +/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into +/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`). +pub async fn put_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, + body: &BTreeMap, +) -> anyhow::Result<()> { + let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid); + let resp = http.put(&url).json(body).send().await?; + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("meta-core PUT {url} returned {status}: {text}"); + } + Ok(()) +} + +/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`). +#[derive(Debug, Deserialize)] +struct MetaGetResponse { + #[serde(default)] + metadata: Option>, +} + +/// `GET /meta/{cid}` — read the current flat field map for a record, or `None` +/// when meta-core has no record yet (404). Used to read back a plugin's merged +/// fields (e.g. filename-parser's `originalTitle`/`movieYear`) before handing +/// them to the next plugin in the pipeline. +pub async fn get_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, +) -> anyhow::Result>> { + let url = format!("{}/meta/{}", base_url.trim_end_matches('/'), cid); + let resp = http.get(&url).send().await?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !resp.status().is_success() { + anyhow::bail!("meta-core GET {url} returned {}", resp.status()); + } + let parsed: MetaGetResponse = resp.json().await?; + Ok(parsed.metadata) +} + +/// Self-publishing store for feeders that own their record lifecycle (D4): +/// the feeder writes its own **metadata-only** records to meta-core and returns +/// sparse outcomes so the gateway dispatcher skips them; the meta-share ingester +/// Kamilata-propagates them. +/// +/// **Byte outcomes are deliberately NOT self-published.** A feeder-side WebDAV +/// write to the gateway core would be seeded by nobody: the gateway core runs +/// with `ENABLE_FILE_WATCHER=false` (so meta-core emits no `/api/events/files` +/// for it, and the ingester's `ipfs_seed` — which seeds only watched files — +/// never sees it), and a sparse outcome makes the gateway skip its own +/// blockstore seed. So byte-bearing outcomes (torznab full-file / subtitles) +/// pass through unchanged and the **gateway** continues to WebDAV-PUT + bitswap- +/// seed them (it's the libp2p/bitswap host). Result: feeder owns the metadata +/// records (the Kamilata-propagated bulk), gateway owns byte delivery — a clean +/// split with no seeding regression. Tribler is metadata-only and uses +/// [`crate::enrich::Enricher`]; this is torznab's record-owning counterpart. +#[derive(Clone)] +pub struct FeederStore { + http: reqwest::Client, + meta_core_url: String, + gateway_peer: String, +} + +impl FeederStore { + pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self { + FeederStore { + http, + meta_core_url: meta_core_url.trim_end_matches('/').to_string(), + gateway_peer, + } + } + + /// Build from the feeder env. `None` when `META_CORE_URL` is unset (the + /// feeder then falls back to the legacy "return outcomes, gateway stores" + /// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`. + pub fn from_env() -> Option { + let meta_core_url = env_nonempty("META_CORE_URL")?; + let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()); + let http = crate::common::build_http_client(30, "meta-feeder-store", None); + Some(Self::new(http, meta_core_url, gateway_peer)) + } + + pub fn meta_core_url(&self) -> &str { + &self.meta_core_url + } + + /// Self-publish a **metadata-only** outcome to meta-core, returning the + /// **sparse** outcome (`record: None`) so the gateway's `(_, None)` branch + /// skips it — no double write. Byte outcomes and already-sparse outcomes are + /// returned unchanged (the gateway stores + seeds the bytes; see the struct + /// docs). On a write failure the original outcome is returned so the gateway + /// stores it (graceful degradation, never lost). + pub async fn publish( + &self, + upstream_id: &str, + record_id: &str, + outcome: HashOutcome, + ) -> HashOutcome { + // Only self-publish metadata-only outcomes (bytes None, record Some). + if outcome.bytes.is_some() || outcome.record.is_none() { + return outcome; + } + let cid = outcome.hash.0.clone(); + let record = outcome.record.as_ref().expect("record is Some by the guard"); + match self + .store_metadata_only(upstream_id, record_id, &cid, record) + .await + { + Ok(()) => sparse(outcome.hash, outcome.hash_kind), + Err(e) => { + warn!(target: "meta-feeder::store", cid = %cid, error = %e, + "metadata self-publish failed; falling back to gateway store"); + outcome + } + } + } + + /// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance. + async fn store_metadata_only( + &self, + upstream_id: &str, + record_id: &str, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer); + let body = build_metadata_body(record, &prov); + put_record(&self.http, &self.meta_core_url, cid, &body).await + } +} + +fn sparse(hash: Hash, hash_kind: HashKind) -> HashOutcome { + HashOutcome { + hash, + hash_kind, + bytes: None, + record: None, + file_extension: None, + } +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rec() -> DiscoveryRecord { + DiscoveryRecord { + upstream_id: "tribler".into(), + record_id: "abc".into(), + fields: BTreeMap::from_iter([ + ("title".into(), "Big Buck Bunny".into()), + ("cid_btih_v1_file".into(), "bafkPERFILE".into()), + ]), + } + } + + #[test] + fn provenance_shape_matches_gateway() { + let p = Provenance { + upstream: "tribler".into(), + record_id: "abc".into(), + gateway_peer: "metagateway-tribler".into(), + acquired_at: 1_747_504_000, + }; + let v: serde_json::Value = serde_json::from_str(&provenance_to_json(&p)).unwrap(); + assert_eq!(v["source"], "gateway"); + assert_eq!(v["upstream"], "tribler"); + assert_eq!(v["recordId"], "abc"); + assert_eq!(v["gatewayPeer"], "metagateway-tribler"); + assert_eq!(v["acquiredAt"], 1_747_504_000_u64); + } + + #[test] + fn body_collapses_cid_fields_and_stamps_provenance() { + let p = Provenance::now("tribler", "abc", "peer"); + let body = build_metadata_body(&rec(), &p); + assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny")); + // cid_* collapsed to cids/=true; no cid_* survives. + assert!(body.keys().all(|k| !k.starts_with("cid_"))); + assert_eq!( + body.get("cids/bafkPERFILE").map(String::as_str), + Some("true") + ); + assert!(body.contains_key("provenance")); + } +} diff --git a/crates/meta-feeder-sdk/src/plugin.rs b/crates/meta-feeder-sdk/src/plugin.rs new file mode 100644 index 0000000..e6fd722 --- /dev/null +++ b/crates/meta-feeder-sdk/src/plugin.rs @@ -0,0 +1,223 @@ +//! The `FeederPlugin` trait + supporting types. +//! +//! A feeder plugin bridges one external upstream (`gutenberg`, `arxiv`, +//! `torznab`, …) into the MetaMesh network. It **finds** records and +//! **fetches** bytes; it does NOT hash-into-a-blockstore, store back into +//! meta-core, or speak libp2p — those stay in the gateway core. This is why +//! `FeederPlugin` has no `set_blockstore` / `set_tmdb_budget` hooks (the +//! gateway's `GatewayPlugin` carried them): preview-seeding moves to the core, +//! and a feeder that needs a TMDB budget owns it internally. +//! +//! The `serve_feeders` harness ([`crate::serve`]) wraps any set of +//! `FeederPlugin`s in the feeder HTTP contract; the gateway core's +//! `RemoteFeederPlugin` is the matching HTTP client. + +use std::collections::HashMap; +use std::path::Path; + +use async_trait::async_trait; +use futures::stream::BoxStream; + +// Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the +// gateway's original `plugin.rs` re-exported it the same way). +pub use crate::query::GatewayQuery; +use crate::query::GatewaySearchEvent; +use crate::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; + +/// One element of what [`FeederPlugin::compute_outcomes`] returns. +/// +/// The `hash` is the value shipped to the gateway core. The other fields are +/// signals the core's auto-store uses to persist the content into meta-core. +/// The core's three-branch auto-store routes on `(bytes, record)`: +/// - `(Some, Some)` — full store: WebDAV PUT + metadata PUT with `filePath`. +/// - `(None, Some)` — metadata-only PUT (no local bytes; e.g. torznab samples +/// the middle 1 MiB of a multi-GiB video and surfaces only the record). +/// - `(_, None)` — skip. Cache hits land here. +/// +/// `file_extension` MUST be `None` when `bytes` is `None`. +#[derive(Debug)] +pub struct HashOutcome { + pub hash: Hash, + /// Which hash family this CID belongs to. Drives the core's routing — + /// the bitswap blockstore seed only fires for `Sha2_256`. + pub hash_kind: HashKind, + pub bytes: Option, + pub record: Option, + /// File extension to append when naming the WebDAV-side blob (e.g. + /// `"epub"`). No leading dot. MUST be `None` when `bytes` is `None`. + pub file_extension: Option, +} + +/// Hash family discriminator on [`HashOutcome`]. +/// +/// - `Midhash256` — size-prefix-plus-middle-1MB-sample, custom multicodec +/// `0x1000`. Fast, MetaMesh-internal, not retrievable via public IPFS. +/// - `Sha2_256` — standard IPFS CIDv1 over full bytes. Retrievable via bitswap +/// once the bytes are in the core's blockstore. What every shipping plugin +/// today emits. +/// - `BtV1File` — a single file inside a BitTorrent v1 torrent, custom +/// multicodec `0x1001`. Locator, not content hash: opaque to bitswap. +/// - `NzbRelease` — a Usenet release (a Newznab listing), custom multicodec +/// `nzb-release` `0x1005`. Self-describing locator: the cid embeds `{host, +/// id}` in an identity multihash, opaque to bitswap and redeemed to bytes +/// only by a credentialed meta-share peer, which decodes the locator from the +/// cid and grabs the `.nzb` via `t=get` — no KV side-table. +/// - `CardLocator` — the identity of a *work* (a series, a film) as published by +/// a metadata bridge, custom multicodec `0x1007`. The only family with **no +/// bytes behind it at all**: the record IS the payload. Always paired with +/// `bytes: None`, so it lands in the `(None, Some)` metadata-only branch +/// above. Opaque to bitswap, and deliberately so — there is nothing to seed. +/// - `NzbPosting` — a Usenet posting we **scanned ourselves**, custom multicodec +/// `nzb-posting` `0x1003`. A content-derived digest over the article +/// Message-ID set (not a locator: no host is embedded), so any peer with a +/// plain NNTP provider can fetch it — no `IndexerCred`. Opaque to bitswap: the +/// digest is over the id set, not over any block, so the `.nzb` manifest +/// travels separately as an ordinary sha2-256 cid in the record's `manifest` +/// field. Produced by `hash::compute_nzb_posting_cid`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashKind { + /// Reserved: no feeder currently emits midhash256 outcomes, but the family + /// is real across meta-core / meta-share, so the variant stays. + #[allow(dead_code)] + Midhash256, + Sha2_256, + BtV1File, + NzbRelease, + CardLocator, + NzbPosting, +} + +/// Startup-only configuration error returned by [`FeederPlugin::configure`]. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("plugin {plugin}: required config missing: {what}")] + MissingConfig { + plugin: &'static str, + /// Human-readable description of the missing setting and where to + /// supply it. Used by the harness's soft-skip warning. + what: &'static str, + }, + + #[error("plugin {plugin}: cache dir setup failed: {source}")] + CacheSetup { + plugin: &'static str, + #[source] + source: std::io::Error, + }, + + #[error("plugin {plugin}: {source}")] + Other { + plugin: &'static str, + #[source] + source: anyhow::Error, + }, +} + +/// Canonical field name for an upstream's stable per-record identifier. +/// Every plugin's `DiscoveryRecord.fields` MUST include this key +/// (`gutenbergid:11`, `tmdbid:31910`, `arxivid:2106.07447`). The rule is +/// `format!("{upstream_id}id")` — lowercase, no separator. +pub fn upstream_id_field(upstream_id: &str) -> String { + format!("{upstream_id}id") +} + +/// Static plugin contract. Each enabled `upstream_id` is implemented by +/// exactly one `Box`. Lives behind `&self` in steady state — +/// called concurrently, owns any interior mutability for caches. +/// +/// **Field-naming convention.** Every record a plugin returns MUST include the +/// canonical `id` field — see [`upstream_id_field`]. +#[async_trait] +pub trait FeederPlugin: Send + Sync + 'static { + fn upstream_id(&self) -> &'static str; + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError>; + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError>; + + /// Streaming variant of [`handle_query`]. The default impl collects + /// [`handle_query`] and replays it as `Base*` + `Done` — correct for every + /// plugin with no incremental enrichment (gutenberg, arxiv, …). torznab + /// overrides it to decouple base records from rate-limited TMDB enrichment. + async fn handle_query_stream( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + let records = self.handle_query(query, max_results).await?; + let events = records + .into_iter() + .map(GatewaySearchEvent::Base) + .chain(std::iter::once(GatewaySearchEvent::Done)); + Ok(Box::pin(futures::stream::iter(events))) + } + + /// Resolve `record_id` into one or more content-addressed outcomes. + /// + /// - `Err(_)` — the upstream record itself is unresolvable. + /// - `Ok(vec![])` — resolved but yielded zero outcomes. + /// - `Ok(partial_vec)` — bundle-success: per-sibling failures are + /// best-effort (log + drop), surface whatever succeeded. + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError>; + + async fn handle_fetch(&self, _record_id: &str) -> Result, GatewayError> { + Ok(None) + } + + fn health(&self) -> PluginHealth { + PluginHealth::Ok + } + + fn served_file_types(&self) -> &'static [&'static str] { + &[] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &[] + } + + /// Self-describing config schema. The feeder SDK serves this at + /// `GET /config/schema` and renders a generic form from it — the gateway and + /// UI carry no per-plugin field knowledge. Default: no configuration. + fn config_schema(&self) -> crate::config::ConfigSchema { + crate::config::ConfigSchema::default() + } + + /// The plugin's current *effective* config as JSON, **unredacted** (secrets + /// included). The SDK redacts it before serving `GET /config/values` and uses + /// it as the merge base on the first save (before any `config.json` exists). + /// Default: empty object. Implementors should mirror the keys in + /// [`config_schema`](FeederPlugin::config_schema). + fn config_values(&self) -> serde_json::Value { + serde_json::json!({}) + } + + async fn get_blob(&self, _cid: &str) -> Option> { + None + } +} + +/// Static plugin registry. Built once at startup; read-only thereafter. +/// Keyed by `upstream_id`. +pub type PluginRegistry = HashMap<&'static str, Box>; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn config_error_messages_include_plugin_name() { + let e = ConfigError::MissingConfig { + plugin: "scihub", + what: + "mirrors (set them in the dashboard or gateway-config.json plugins.scihub.mirrors)", + }; + let msg = e.to_string(); + assert!(msg.contains("scihub")); + assert!(msg.contains("mirrors")); + } +} diff --git a/crates/meta-feeder-sdk/src/query.rs b/crates/meta-feeder-sdk/src/query.rs new file mode 100644 index 0000000..00f54b8 --- /dev/null +++ b/crates/meta-feeder-sdk/src/query.rs @@ -0,0 +1,147 @@ +//! Structured query envelope + streaming-search event + wire-error types, +//! shared by feeders (their HTTP contract) and the gateway core (its libp2p +//! wire). These are pure serde data types — no libp2p, no transport. +//! +//! On the core side, `protocol.rs` re-uses these exact structs as the bincode +//! wire shape; the `wire_parity_*` tests there pin the byte encoding. Keep +//! field order / variant order / derive shape stable. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::types::{DiscoveryRecord, GatewayError}; + +/// Structured query envelope. Gateway plugins do best-effort pushdown to +/// upstream APIs with native query languages (`torznab cat=movie`, +/// `arxiv id_list=`). +/// +/// **Authority rule:** `filters` / `ranges` / `negations` are authoritative; +/// `raw_text` is informational. A plugin that consumes both will double-filter +/// and under-match — pick one source per filter; structured wins. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GatewayQuery { + /// Original user text (e.g. `"naruto contentKind:movie imdbid:tt0903747"`). + /// Verbatim — never mutated by the conversion. + pub raw_text: String, + + /// Bare-word leaves only, space-joined. The plugin's primary `q=` input + /// for upstream search APIs that take a single search box. + pub free_text: String, + + /// Structured filters keyed by field name (raw, unstemmed values). + /// Multiple values per key = OR within a key; AND across keys. + pub filters: BTreeMap>, + + /// Numeric range filters (e.g. `movieYear:2020..2024`). Both bounds + /// optional → unbounded on that side. + pub ranges: Vec, + + /// Negation filters (e.g. `NOT genres:horror`, bare `NOT word`). + pub negations: Vec, +} + +/// Numeric range filter inside [`GatewayQuery::ranges`]. Both bounds are +/// `Option` so open-ended ranges drop one bound to `None`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RangeFilter { + pub field: String, + pub lo: Option, + pub hi: Option, +} + +/// Negation entry inside [`GatewayQuery::negations`]. `field` is `Some(name)` +/// for typed negations (`NOT genres:horror`), `None` for bare-word negation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Negation { + pub field: Option, + pub value: String, +} + +impl GatewayQuery { + /// Bare-text input to hand to a free-form upstream search API. Falls back + /// to `"*"` when the conversion produced no free-text leaves so upstreams + /// that 4xx on empty search return their top-ranked results instead. + pub fn free_text_or_star(&self) -> &str { + if self.free_text.trim().is_empty() { + "*" + } else { + self.free_text.as_str() + } + } + + /// Construct a free-text-only `GatewayQuery` for in-process use (tests, + /// debug routes). `raw_text` == `free_text`; structured fields stay empty. + pub fn from_free_text(text: impl Into) -> Self { + let text = text.into(); + Self { + raw_text: text.clone(), + free_text: text, + filters: BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + } + } +} + +/// One frame in the streaming Search response. +/// +/// A plugin emits `Base` records as soon as they're discovered, then +/// best-effort `EnrichPatch`/`Drop` events as enrichment lands, then a +/// terminal `Done`. The default `handle_query_stream` collects `handle_query` +/// and replays it as `Base*` + `Done`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum GatewaySearchEvent { + /// An un-enriched discovery record. + Base(DiscoveryRecord), + /// Incremental enrichment for a previously-sent `Base`, keyed by + /// `record_id`. `set` fields are inserted/overwritten; `remove` deleted. + EnrichPatch { + record_id: String, + set: BTreeMap, + remove: Vec, + }, + /// Retract a previously-emitted `Base` record entirely. + Drop { record_id: String }, + /// Terminal success — no more frames follow. + Done, + /// Terminal error — no more frames follow. + Error(GatewayWireError), +} + +/// Wire-stable error shape. Distinct from in-process [`GatewayError`] because +/// adding a variant here is a wire-format change. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] +pub enum GatewayWireError { + #[error("not found")] + NotFound, + #[error("rate limited; retry in {retry_after_s}s")] + RateLimited { retry_after_s: u32 }, + #[error("upstream transient: {0}")] + UpstreamTransient(String), + #[error("upstream permanent: {0}")] + UpstreamPermanent(String), + /// The protocol was registered but no plugin claims this `upstream_id`. + #[error("plugin not loaded")] + PluginNotLoaded, + /// Internal codec / decode failure on the *receiver* side. + #[error("internal: {0}")] + Internal(String), +} + +impl From for GatewayWireError { + fn from(e: GatewayError) -> Self { + match e { + GatewayError::Transient(s) => GatewayWireError::UpstreamTransient(s), + GatewayError::Permanent(s) => GatewayWireError::UpstreamPermanent(s), + GatewayError::RateLimited { retry_after_s } => { + GatewayWireError::RateLimited { retry_after_s } + } + GatewayError::NotFound => GatewayWireError::NotFound, + // Internal errors are never leaked; map to a transient. + GatewayError::Internal(_) => { + GatewayWireError::UpstreamTransient("internal".to_string()) + } + } + } +} diff --git a/crates/meta-feeder-sdk/src/query_eval.rs b/crates/meta-feeder-sdk/src/query_eval.rs new file mode 100644 index 0000000..523187d --- /dev/null +++ b/crates/meta-feeder-sdk/src/query_eval.rs @@ -0,0 +1,456 @@ +//! Structured-filter evaluation for gateway records. +//! +//! Two pure helpers consuming the [`crate::query::GatewayQuery`] struct: +//! +//! - [`record_matches`] — does a single record's `fields` map satisfy the +//! query's `filters` + `ranges` + typed `negations`? +//! - [`query_accepts_plugin`] — could a plugin advertising the given +//! `served_file_types` / `served_content_kinds` possibly produce a record +//! that satisfies the query? Used by plugins as a cheap early-return. +//! +//! The dispatcher applies `record_matches` to every plugin's return value, and +//! meta-share applies the same algorithm consumer-side to defend against stale +//! gateway peers — both tiers must agree on outcomes for any (record, query) +//! pair. See the original gateway crate docs for the full semantics. + +use std::collections::BTreeMap; + +use crate::query::GatewayQuery; + +/// True iff a record's `fields` satisfies every filter, range, and typed +/// negation in `query`. +pub fn record_matches(fields: &BTreeMap, query: &GatewayQuery) -> bool { + for (key, allowed) in &query.filters { + if key == "languages" { + if !languages_filter_matches(fields, allowed) { + return false; + } + continue; + } + let Some(field_value) = fields.get(key) else { + return false; + }; + if !any_csv_value_matches(field_value, allowed) { + return false; + } + } + + for range in &query.ranges { + let Some(field_value) = fields.get(&range.field) else { + return false; + }; + let Ok(n) = field_value.parse::() else { + return false; + }; + if let Some(lo) = range.lo { + if n < lo { + return false; + } + } + if let Some(hi) = range.hi { + if n > hi { + return false; + } + } + } + + for negation in &query.negations { + let Some(name) = negation.field.as_deref() else { + // Bare-word negation — not enforced here. + continue; + }; + let Some(field_value) = fields.get(name) else { + // Missing field ⇒ negation passes (nothing to exclude). + continue; + }; + if any_csv_value_matches(field_value, std::slice::from_ref(&negation.value)) { + return false; + } + } + + true +} + +/// True iff a plugin advertising the given `served_file_types` / +/// `served_content_kinds` could possibly produce a record satisfying +/// `query.filters["fileType" | "contentKind"]`. +pub fn query_accepts_plugin( + query: &GatewayQuery, + served_file_types: &[&str], + served_content_kinds: &[&str], +) -> bool { + accepts_axis(query.filters.get("fileType"), served_file_types) + && accepts_axis(query.filters.get("contentKind"), served_content_kinds) +} + +fn accepts_axis(requested: Option<&Vec>, served: &[&str]) -> bool { + let Some(requested) = requested else { + return true; + }; + if requested.is_empty() { + return true; + } + // Wildcard: a plugin that advertises `"*"` on an axis serves every value on + // that axis (e.g. tribler is a free-text overlay search with no inherent + // type limitation). It accepts any requested fileType/contentKind. + if served.contains(&"*") { + return true; + } + requested + .iter() + .any(|v| served.iter().any(|s| s.eq_ignore_ascii_case(v))) +} + +/// Resolve a `languages` filter against the key-set storage form +/// (`languages/ = "true"`). **Fails open** on anything we can't +/// confidently exclude. A record matches when: +/// 1. it carries a requested code as a truthy key-set member (OR across codes); +/// 2. a legacy flat `languages` csv field contains a requested code; +/// 3. it carries `mul` (multiple — may contain a requested one) or `und` +/// (undetermined); or +/// 4. it carries no language information at all. +/// +/// Only a record with a *concrete*, non-matching language (e.g. `jpn`-only +/// under an `eng` filter) is dropped. Kept byte-identical in logic with +/// meta-search's copy (and the client's `passesLangFilter`) — language narrows +/// the result set without ever hiding unknown/undetermined/multi releases. +fn languages_filter_matches(fields: &BTreeMap, allowed: &[String]) -> bool { + for code in allowed { + let key = format!("languages/{}", code.trim().to_ascii_lowercase()); + if fields.get(&key).is_some_and(|v| is_truthy_member(v)) { + return true; + } + } + if let Some(flat) = fields.get("languages") { + if any_csv_value_matches(flat, allowed) { + return true; + } + } + // Fail open: `mul`/`und` are explicit "could be anything / unknown" markers, + // and a record with no concrete language at all is equally unknown. Keep all + // three; only a specific, non-matching language reaches the `false` below. + if fields.get("languages/mul").is_some_and(|v| is_truthy_member(v)) + || fields.get("languages/und").is_some_and(|v| is_truthy_member(v)) + { + return true; + } + !record_has_concrete_language(fields) +} + +/// True iff the record states a *concrete* language: a truthy `languages/` +/// key-set member other than the `und`/`mul` non-codes, or a non-empty legacy +/// flat `languages` field. Distinguishes "a specific language that didn't match +/// ⇒ drop" from "no language info ⇒ keep" for [`languages_filter_matches`]. +fn record_has_concrete_language(fields: &BTreeMap) -> bool { + if fields.get("languages").is_some_and(|v| !v.trim().is_empty()) { + return true; + } + fields.iter().any(|(k, v)| { + is_truthy_member(v) + && k.starts_with("languages/") + && k != "languages/und" + && k != "languages/mul" + }) +} + +/// Key-set members are written as the literal `"true"`; tolerate a couple of +/// obvious truthy spellings and treat anything else as absent. +fn is_truthy_member(v: &str) -> bool { + matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes") +} + +/// True iff `field_value` (treated as a csv set when it contains commas) +/// contains at least one case-insensitive match for any item in `allowed`. +fn any_csv_value_matches(field_value: &str, allowed: &[String]) -> bool { + for piece in field_value.split(',') { + let piece = piece.trim(); + if piece.is_empty() { + continue; + } + if allowed.iter().any(|v| v.eq_ignore_ascii_case(piece)) { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::query::{Negation, RangeFilter}; + + fn empty_query() -> GatewayQuery { + GatewayQuery { + raw_text: String::new(), + free_text: String::new(), + filters: BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + } + } + + fn with_filter(field: &str, values: &[&str]) -> GatewayQuery { + let mut q = empty_query(); + q.filters.insert( + field.to_string(), + values.iter().map(|s| s.to_string()).collect(), + ); + q + } + + fn fields(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn filter_rejects_mismatched_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + } + + #[test] + fn filter_accepts_matching_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + } + + #[test] + fn filter_rejects_record_missing_field() { + let q = with_filter("fileType", &["video"]); + assert!(!record_matches(&fields(&[]), &q)); + } + + #[test] + fn filter_is_case_insensitive() { + let q = with_filter("fileType", &["VIDEO"]); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + let q = with_filter("fileType", &["video"]); + assert!(record_matches(&fields(&[("fileType", "VIDEO")]), &q)); + } + + #[test] + fn filter_ors_within_key() { + let q = with_filter("fileType", &["video", "audio"]); + assert!(record_matches(&fields(&[("fileType", "audio")]), &q)); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + } + + #[test] + fn filter_ands_across_keys() { + let mut q = with_filter("fileType", &["video"]); + q.filters.insert("genres".into(), vec!["Action".into()]); + assert!(!record_matches( + &fields(&[("fileType", "video"), ("genres", "Drama")]), + &q + )); + assert!(record_matches( + &fields(&[("fileType", "video"), ("genres", "Action")]), + &q + )); + } + + #[test] + fn csv_field_matches_one_of_its_values() { + let q = with_filter("genres", &["Anime"]); + assert!(record_matches( + &fields(&[("genres", "Animation,Anime,Drama")]), + &q + )); + } + + #[test] + fn csv_field_with_whitespace_is_trimmed() { + let q = with_filter("genres", &["Anime"]); + assert!(record_matches( + &fields(&[("genres", "Animation, Anime , Drama")]), + &q + )); + } + + #[test] + fn languages_keyset_member_matches() { + let q = with_filter("languages", &["eng"]); + assert!(record_matches(&fields(&[("languages/eng", "true")]), &q)); + } + + #[test] + fn languages_filter_is_or_across_codes() { + let q = with_filter("languages", &["eng", "fre"]); + assert!(record_matches(&fields(&[("languages/eng", "true")]), &q)); + assert!(record_matches(&fields(&[("languages/fre", "true")]), &q)); + assert!(!record_matches(&fields(&[("languages/jpn", "true")]), &q)); + } + + #[test] + fn languages_filter_fails_open_on_unknown() { + let q = with_filter("languages", &["eng"]); + // No language info at all ⇒ kept (unknown, not excluded). + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + // `mul` (multi) may contain the requested language ⇒ kept. + assert!(record_matches(&fields(&[("languages/mul", "true")]), &q)); + // `und` (undetermined) ⇒ kept. + assert!(record_matches(&fields(&[("languages/und", "true")]), &q)); + // A specific language alongside `mul` is still kept (multi). + assert!(record_matches( + &fields(&[("languages/jpn", "true"), ("languages/mul", "true")]), + &q + )); + } + + #[test] + fn languages_concrete_mismatch_still_drops() { + let q = with_filter("languages", &["eng"]); + // A concrete, non-matching language ⇒ dropped (language narrows). + assert!(!record_matches(&fields(&[("languages/jpn", "true")]), &q)); + // A tombstoned/false member is treated as absent ⇒ unknown ⇒ kept. + assert!(record_matches(&fields(&[("languages/eng", "false")]), &q)); + } + + #[test] + fn languages_legacy_flat_csv_fallback() { + let q = with_filter("languages", &["eng"]); + assert!(record_matches(&fields(&[("languages", "jpn,eng")]), &q)); + assert!(!record_matches(&fields(&[("languages", "jpn,ger")]), &q)); + } + + #[test] + fn range_in_bounds() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: Some(2020), + }); + assert!(record_matches(&fields(&[("movieYear", "2015")]), &q)); + } + + #[test] + fn range_out_of_bounds_rejected() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: Some(2020), + }); + assert!(!record_matches(&fields(&[("movieYear", "2025")]), &q)); + assert!(!record_matches(&fields(&[("movieYear", "2005")]), &q)); + } + + #[test] + fn range_open_upper_bound() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: None, + }); + assert!(record_matches(&fields(&[("movieYear", "2025")]), &q)); + } + + #[test] + fn range_unparseable_value_rejected() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: None, + }); + assert!(!record_matches(&fields(&[("movieYear", "circa 2015")]), &q)); + } + + #[test] + fn typed_negation_excludes_match() { + let mut q = empty_query(); + q.negations.push(Negation { + field: Some("fileType".into()), + value: "image".into(), + }); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + } + + #[test] + fn typed_negation_passes_when_field_missing() { + let mut q = empty_query(); + q.negations.push(Negation { + field: Some("fileType".into()), + value: "image".into(), + }); + assert!(record_matches(&fields(&[]), &q)); + } + + #[test] + fn bare_word_negation_is_not_enforced() { + let mut q = empty_query(); + q.negations.push(Negation { + field: None, + value: "alice".into(), + }); + assert!(record_matches( + &fields(&[("title", "alice in wonderland")]), + &q + )); + } + + #[test] + fn empty_query_matches_anything() { + assert!(record_matches(&fields(&[]), &empty_query())); + assert!(record_matches( + &fields(&[("fileType", "video"), ("title", "x")]), + &empty_query() + )); + } + + #[test] + fn accepts_returns_true_when_no_type_filter() { + let q = empty_query(); + assert!(query_accepts_plugin(&q, &["video"], &["movie"])); + assert!(query_accepts_plugin(&q, &[], &[])); + } + + #[test] + fn accepts_rejects_disjoint_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(!query_accepts_plugin(&q, &["image"], &["gif"])); + assert!(query_accepts_plugin(&q, &["video"], &[])); + } + + #[test] + fn accepts_rejects_disjoint_contentkind() { + let q = with_filter("contentKind", &["movie"]); + assert!(!query_accepts_plugin( + &q, + &["video", "image"], + &["paper", "book"] + )); + assert!(query_accepts_plugin(&q, &["video"], &["movie", "episode"])); + } + + #[test] + fn accepts_requires_both_axes_when_both_filtered() { + let mut q = with_filter("fileType", &["video"]); + q.filters.insert("contentKind".into(), vec!["movie".into()]); + assert!(query_accepts_plugin(&q, &["video"], &["movie"])); + assert!(!query_accepts_plugin(&q, &["video"], &["paper"])); + assert!(!query_accepts_plugin(&q, &["image"], &["movie"])); + } + + #[test] + fn accepts_wildcard_matches_any_requested_type() { + // A plugin advertising "*" (e.g. tribler) accepts any fileType / + // contentKind filter on the wildcarded axis. + let q = with_filter("fileType", &["archive"]); + assert!(query_accepts_plugin(&q, &["*"], &[])); + let q = with_filter("contentKind", &["audiobook"]); + assert!(query_accepts_plugin(&q, &["*"], &["*"])); + // Wildcard on one axis only: the non-wildcard axis still gates. + let mut q = with_filter("fileType", &["audio"]); + q.filters.insert("contentKind".into(), vec!["podcast".into()]); + assert!(query_accepts_plugin(&q, &["*"], &["*"])); + assert!(!query_accepts_plugin(&q, &["*"], &["movie"])); + } +} diff --git a/crates/meta-feeder-sdk/src/serve.rs b/crates/meta-feeder-sdk/src/serve.rs new file mode 100644 index 0000000..447650a --- /dev/null +++ b/crates/meta-feeder-sdk/src/serve.rs @@ -0,0 +1,759 @@ +//! `serve_feeders` — the axum harness that exposes a set of [`FeederPlugin`]s +//! over the **feeder HTTP contract**. Every feeder sidecar binary is just +//! `serve_feeders(vec![MyPlugin::new()], state_dir, listen)`. +//! +//! # Contract (JSON over HTTP, mirrors meta-sort's container-plugin shape) +//! +//! - `GET /manifest` → [`ManifestResponse`] +//! - `GET /health` → [`HealthResponse`] +//! - `POST /query` → [`QueryRequest`] → [`QueryResponse`] +//! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`] +//! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404) +//! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404) +//! +//! ## Bytes & the v1 simplification +//! +//! `/compute` returns each outcome's bytes **inline, base64-encoded** +//! ([`OutcomeDto::bytes_b64`]) — this reuses every existing `compute_outcomes` +//! impl verbatim (they already return bytes in `HashOutcome.bytes`) and keeps +//! the gateway core's auto-store a single round-trip. base64 adds ~33%; fine +//! for books/papers/posters. Large-file streaming (torznab multi-GiB) is a +//! documented follow-up — those outcomes are metadata-only anyway +//! (`bytes: None`), so they never ride this path. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::path::{Path as FsPath, PathBuf}; +use std::sync::Arc; + +use axum::{ + body::Body, + extract::{Path, State}, + http::StatusCode, + response::{Html, IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use base64::Engine as _; +use futures::StreamExt; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{info, warn}; + +use crate::config::{self, CONFIG_PAGE_HTML}; +use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; +use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError}; +use crate::types::{DiscoveryRecord, GatewayError, PluginHealth}; + +// -------- wire DTOs -------------------------------------------------------- + +/// Serializable mirror of [`HashKind`] for the feeder HTTP boundary. +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +#[serde(rename_all = "snake_case")] +pub enum HashKindDto { + Midhash256, + Sha2_256, + BtV1File, + NzbRelease, + CardLocator, + NzbPosting, +} + +impl From for HashKindDto { + fn from(k: HashKind) -> Self { + match k { + HashKind::Midhash256 => HashKindDto::Midhash256, + HashKind::Sha2_256 => HashKindDto::Sha2_256, + HashKind::BtV1File => HashKindDto::BtV1File, + HashKind::NzbRelease => HashKindDto::NzbRelease, + HashKind::CardLocator => HashKindDto::CardLocator, + HashKind::NzbPosting => HashKindDto::NzbPosting, + } + } +} + +impl From for HashKind { + fn from(k: HashKindDto) -> Self { + match k { + HashKindDto::Midhash256 => HashKind::Midhash256, + HashKindDto::Sha2_256 => HashKind::Sha2_256, + HashKindDto::BtV1File => HashKind::BtV1File, + HashKindDto::NzbRelease => HashKind::NzbRelease, + HashKindDto::CardLocator => HashKind::CardLocator, + HashKindDto::NzbPosting => HashKind::NzbPosting, + } + } +} + +/// One plugin's capability surface in [`ManifestResponse`]. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct PluginManifest { + pub id: String, + pub served_file_types: Vec, + pub served_content_kinds: Vec, +} + +/// `GET /manifest` — the feeder's hosted plugins + their served types. The +/// gateway core reads this to register `RemoteFeederPlugin`s and to build the +/// capability heartbeat it broadcasts to meta-share. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ManifestResponse { + pub feeder_version: String, + pub plugins: Vec, +} + +/// One plugin's liveness in [`HealthResponse`]. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct PluginHealthEntry { + pub id: String, + pub health: PluginHealth, +} + +/// `GET /health` — `status` is `"ok"` iff every hosted plugin reports `Ok`. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct HealthResponse { + pub status: String, + pub plugins: Vec, +} + +/// `POST /query` request body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct QueryRequest { + pub upstream_id: String, + pub query: GatewayQuery, + pub max_results: u32, +} + +/// `POST /query` response body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct QueryResponse { + pub records: Vec, +} + +/// `POST /compute` request body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ComputeRequest { + pub upstream_id: String, + pub record_id: String, +} + +/// One resolved outcome on the feeder wire. `bytes_b64` carries the full +/// payload base64-encoded when the plugin fetched bytes (`Sha2_256` full +/// store); `None` for metadata-only outcomes (the core's `(None, Some)` +/// branch — e.g. torznab midhash). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutcomeDto { + pub hash: String, + pub hash_kind: HashKindDto, + pub file_extension: Option, + pub record: Option, + pub bytes_b64: Option, +} + +impl OutcomeDto { + fn from_outcome(o: HashOutcome) -> Self { + OutcomeDto { + hash: o.hash.0, + hash_kind: o.hash_kind.into(), + file_extension: o.file_extension, + record: o.record, + bytes_b64: o + .bytes + .map(|b| base64::engine::general_purpose::STANDARD.encode(&b)), + } + } +} + +/// `POST /compute` response body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ComputeResponse { + pub outcomes: Vec, +} + +// -------- app state + error mapping ---------------------------------------- + +#[derive(Clone)] +struct AppState { + plugins: Arc>>, + version: String, + /// Feeder state dir. Per-plugin config persists at + /// `state_dir/gateway//config.json` — the same per-plugin dir handed to + /// `configure()`, so a UI save is read back on the next restart. + state_dir: PathBuf, +} + +impl AppState { + fn plugin(&self, upstream_id: &str) -> Result<&Arc, Response> { + self.plugins.get(upstream_id).ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + format!("no feeder plugin for upstream_id '{upstream_id}'"), + ) + .into_response() + }) + } + + /// `state_dir/gateway//config.json` — canonical per-plugin config file. + fn config_path(&self, upstream_id: &str) -> PathBuf { + self.state_dir + .join("gateway") + .join(upstream_id) + .join("config.json") + } + + /// The effective config as JSON: the persisted `config.json` if it exists, + /// else the plugin's in-memory (env-seeded) config. Used as the redaction + /// source for reads and the merge base for writes. + fn effective_config(&self, upstream_id: &str) -> Value { + if let Ok(bytes) = std::fs::read(self.config_path(upstream_id)) { + if let Ok(v) = serde_json::from_slice::(&bytes) { + return v; + } + } + self.plugins + .get(upstream_id) + .map(|p| p.config_values()) + .unwrap_or_else(|| serde_json::json!({})) + } +} + +/// Map an in-process [`GatewayError`] to an HTTP status + message. Mirrors the +/// status semantics the gateway's `RemoteFeederPlugin` will translate back +/// into a `GatewayError` on the core side. +fn gateway_error_response(e: GatewayError) -> Response { + let (status, msg) = match e { + GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()), + GatewayError::RateLimited { retry_after_s } => ( + StatusCode::TOO_MANY_REQUESTS, + format!("rate limited; retry in {retry_after_s}s"), + ), + GatewayError::Transient(s) => (StatusCode::BAD_GATEWAY, s), + GatewayError::Permanent(s) => (StatusCode::UNPROCESSABLE_ENTITY, s), + GatewayError::Internal(s) => (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()), + }; + (status, msg).into_response() +} + +// -------- handlers --------------------------------------------------------- + +async fn manifest(State(state): State) -> Json { + let mut plugins: Vec = state + .plugins + .values() + .map(|p| PluginManifest { + id: p.upstream_id().to_string(), + served_file_types: p.served_file_types().iter().map(|s| s.to_string()).collect(), + served_content_kinds: p + .served_content_kinds() + .iter() + .map(|s| s.to_string()) + .collect(), + }) + .collect(); + plugins.sort_by(|a, b| a.id.cmp(&b.id)); + Json(ManifestResponse { + feeder_version: state.version.clone(), + plugins, + }) +} + +async fn health(State(state): State) -> Json { + let mut entries: Vec = state + .plugins + .values() + .map(|p| PluginHealthEntry { + id: p.upstream_id().to_string(), + health: p.health(), + }) + .collect(); + entries.sort_by(|a, b| a.id.cmp(&b.id)); + let all_ok = entries + .iter() + .all(|e| matches!(e.health, PluginHealth::Ok)); + Json(HealthResponse { + status: if all_ok { "ok" } else { "degraded" }.to_string(), + plugins: entries, + }) +} + +async fn query( + State(state): State, + Json(req): Json, +) -> Result, Response> { + let plugin = state.plugin(&req.upstream_id)?; + let records = plugin + .handle_query(&req.query, req.max_results as usize) + .await + .map_err(gateway_error_response)?; + Ok(Json(QueryResponse { records })) +} + +/// `POST /query_stream` — the streaming analog of [`query`]. Emits the plugin's +/// [`FeederPlugin::handle_query_stream`] events as **NDJSON**: one JSON-encoded +/// [`GatewaySearchEvent`] per line (`application/x-ndjson`). The gateway core's +/// `RemoteFeederPlugin` consumes this and re-emits a `BoxStream` +/// into the libp2p streaming worker, preserving incremental enrichment (e.g. +/// torznab's late TMDB / poster / subtitle-availability patches). A plugin that +/// doesn't override `handle_query_stream` still works here — the trait default +/// replays `handle_query` as `Base* + Done`. +async fn query_stream( + State(state): State, + Json(req): Json, +) -> Result { + let plugin = state.plugin(&req.upstream_id)?; + // The open can fail (e.g. the plugin can't reach its upstream at all); that + // maps to an HTTP error status, same as `/query`. Per-event errors instead + // ride the stream as a terminal `GatewaySearchEvent::Error` line. + let events = plugin + .handle_query_stream(&req.query, req.max_results as usize) + .await + .map_err(gateway_error_response)?; + let body = events.map(|ev| { + // Serialization of GatewaySearchEvent is infallible in practice; on the + // off chance it fails, terminate with an Error line so the consumer sees + // a clean terminal failure rather than a truncated stream. + let mut line = serde_json::to_vec(&ev).unwrap_or_else(|e| { + serde_json::to_vec(&GatewaySearchEvent::Error(GatewayWireError::Internal( + format!("event serialize: {e}"), + ))) + .unwrap_or_default() + }); + line.push(b'\n'); + Ok::<_, std::convert::Infallible>(line) + }); + Ok(( + [(axum::http::header::CONTENT_TYPE, "application/x-ndjson")], + Body::from_stream(body), + ) + .into_response()) +} + +async fn compute( + State(state): State, + Json(req): Json, +) -> Result, Response> { + let plugin = state.plugin(&req.upstream_id)?; + let outcomes = plugin + .compute_outcomes(&req.record_id) + .await + .map_err(gateway_error_response)?; + Ok(Json(ComputeResponse { + outcomes: outcomes.into_iter().map(OutcomeDto::from_outcome).collect(), + })) +} + +async fn fetch( + State(state): State, + Path((upstream_id, record_id)): Path<(String, String)>, +) -> Result { + let plugin = state.plugin(&upstream_id)?; + match plugin + .handle_fetch(&record_id) + .await + .map_err(gateway_error_response)? + { + Some(stream) => { + // Map the plugin's GatewayError stream into an io::Error stream so + // axum's Body::from_stream accepts it. + let body_stream = stream.map(|chunk| { + chunk.map_err(|e| std::io::Error::other(e.to_string())) + }); + Ok(Body::from_stream(body_stream).into_response()) + } + None => Err((StatusCode::NOT_FOUND, "no bytes for record").into_response()), + } +} + +async fn blob( + State(state): State, + Path((upstream_id, cid)): Path<(String, String)>, +) -> Result { + let plugin = state.plugin(&upstream_id)?; + match plugin.get_blob(&cid).await { + Some(bytes) => Ok(bytes.into_response()), + None => Err((StatusCode::NOT_FOUND, "no such blob").into_response()), + } +} + +// -------- config plane ----------------------------------------------------- +// +// Generic, schema-driven per-plugin config. The page + the schema/values JSON +// are all served relative to the request path, so this works identically when +// hit directly on the feeder or reverse-proxied through the gateway. + +/// `GET /config` — the schema-driven HTML form (static, plugin-agnostic). +async fn config_page() -> Html<&'static str> { + Html(CONFIG_PAGE_HTML) +} + +/// `GET /config/schema` — the single hosted plugin's declared config schema. +/// (A feeder hosts one plugin in practice; if it hosts several, the first by +/// id is used — the gateway proxies per upstream_id anyway.) +async fn config_schema(State(state): State) -> Json { + let schema = sole_plugin(&state) + .map(|p| p.config_schema()) + .unwrap_or_default(); + Json(schema) +} + +/// `GET /config/values` — current values with secrets redacted to `_set`. +async fn config_values_get(State(state): State) -> Json { + match sole_plugin(&state) { + Some(p) => { + let schema = p.config_schema(); + let eff = state.effective_config(p.upstream_id()); + Json(config::redact(&eff, &schema)) + } + None => Json(serde_json::json!({})), + } +} + +/// `PUT /config/values` — merge the submitted values onto the stored config +/// (schema-aware; blank secrets keep their stored value), persist atomically, +/// and signal that a feeder restart is needed to apply them. +async fn config_values_put( + State(state): State, + Json(incoming): Json, +) -> Result, Response> { + let plugin = sole_plugin(&state) + .ok_or_else(|| (StatusCode::NOT_FOUND, "feeder hosts no plugin").into_response())?; + let id = plugin.upstream_id(); + let schema = plugin.config_schema(); + let base = state.effective_config(id); + let merged = config::merge(&base, &incoming, &schema); + + let path = state.config_path(id); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("mkdir: {e}")).into_response())?; + } + let tmp = path.with_extension("json.tmp"); + let body = serde_json::to_vec_pretty(&merged) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("serialize: {e}")).into_response())?; + std::fs::write(&tmp, body) + .and_then(|_| std::fs::rename(&tmp, &path)) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("write config: {e}")).into_response())?; + info!(target: "meta-feeder", upstream_id = id, path = %path.display(), "feeder config saved; scheduling self-restart to apply"); + // Apply the new config by restarting the process. We can't hot-swap config + // in-place: plugins hold an exclusive redb cache handle for their whole life, + // so a fresh instance can't reopen the cache while the old one is alive. A + // clean process exit is the reliable apply path — Docker's restart policy + // (`unless-stopped`) brings the feeder back in ~2-3s and `configure()` + // re-reads config.json at startup. Delay briefly so THIS response flushes to + // the client before we exit. + tokio::spawn(async { + tokio::time::sleep(std::time::Duration::from_millis(800)).await; + info!(target: "meta-feeder", "restarting now to apply saved config"); + std::process::exit(0); + }); + Ok(Json(serde_json::json!({ "saved": true, "reloading": true }))) +} + +/// `POST /enrich/callback` — sink for the enrichment plugins' completion +/// callbacks. The feeder drives those plugins (filename-parser, tmdb) during a +/// self-published resolve and reads results back by polling meta-core, so the +/// callback carries no load — this route exists only so the plugins' callback +/// POST gets a 200 instead of logging a connection failure. Body is ignored. +async fn enrich_callback() -> StatusCode { + StatusCode::OK +} + +/// A feeder hosts one plugin in the common case. Pick it (lowest id for +/// determinism if several are present). +fn sole_plugin(state: &AppState) -> Option<&Arc> { + state + .plugins + .values() + .min_by(|a, b| a.upstream_id().cmp(b.upstream_id())) +} + +// -------- entry point ------------------------------------------------------ + +/// Build the feeder router for an already-configured plugin set. Exposed for +/// in-process tests (drive it without binding a socket). +pub fn router( + plugins: HashMap>, + version: String, + state_dir: impl Into, +) -> Router { + let state = AppState { + plugins: Arc::new(plugins), + version, + state_dir: state_dir.into(), + }; + Router::new() + .route("/manifest", get(manifest)) + .route("/health", get(health)) + .route("/query", post(query)) + .route("/query_stream", post(query_stream)) + .route("/compute", post(compute)) + .route("/fetch/:upstream_id/:record_id", get(fetch)) + .route("/blob/:upstream_id/:cid", get(blob)) + .route("/config", get(config_page)) + .route("/config/schema", get(config_schema)) + .route("/config/values", get(config_values_get).put(config_values_put)) + .route("/enrich/callback", post(enrich_callback)) + .with_state(state) +} + +/// Configure each plugin (cache dir under `state_dir/gateway/`), then serve +/// the feeder HTTP contract on `listen` until shutdown. +/// +/// The feeder process never aborts on a config problem: a plugin whose +/// `configure()` returns [`ConfigError::MissingConfig`] is **soft-skipped** +/// (warn + dropped — an optional source the operator didn't ask for), and any +/// other failure (bad/empty key, cache-dir setup, …) registers a +/// [`DegradedPlugin`] that stays up and reports the reason via `/health` and +/// `/manifest`. The container keeps running so the error is visible in the UX +/// instead of crash-looping. +pub async fn serve_feeders( + plugins: Vec>, + state_dir: impl Into, + listen: SocketAddr, +) -> anyhow::Result<()> { + let state_dir = state_dir.into(); + let configured = configure_plugins(plugins, &state_dir)?; + let version = env!("CARGO_PKG_VERSION").to_string(); + let app = router(configured, version, state_dir.clone()); + + let listener = tokio::net::TcpListener::bind(listen).await?; + info!( + target: "meta-feeder", + %listen, + "feeder listening" + ); + axum::serve(listener, app) + .await + .map_err(anyhow::Error::from) +} + +/// A plugin whose `configure()` (or cache-dir setup) failed. It stays +/// **registered** so the failure is visible at `GET /health` (as +/// [`PluginHealth::Degraded`]) and `GET /manifest`; any query/compute against it +/// returns the captured reason as a permanent error. This is what keeps one +/// misconfigured feeder from crash-looping the whole container — the operator +/// sees the reason in the UX and fixes the config, no restart-loop required. +struct DegradedPlugin { + id: &'static str, + reason: String, + file_types: &'static [&'static str], + content_kinds: &'static [&'static str], + /// Preserved from the wrapped plugin so a degraded/unconfigured feeder still + /// serves its config form — otherwise the dashboard shows "no configuration + /// required" and the operator can never fix the very config that's missing. + schema: crate::config::ConfigSchema, +} + +#[async_trait::async_trait] +impl FeederPlugin for DegradedPlugin { + fn upstream_id(&self) -> &'static str { + self.id + } + + fn configure(&mut self, _cache_dir: &FsPath) -> Result<(), ConfigError> { + Ok(()) + } + + async fn handle_query( + &self, + _query: &GatewayQuery, + _max_results: usize, + ) -> Result, GatewayError> { + Err(GatewayError::Permanent(self.reason.clone())) + } + + async fn compute_outcomes( + &self, + _record_id: &str, + ) -> Result, GatewayError> { + Err(GatewayError::Permanent(self.reason.clone())) + } + + fn health(&self) -> PluginHealth { + PluginHealth::Degraded { + reason: self.reason.clone(), + } + } + + fn served_file_types(&self) -> &'static [&'static str] { + self.file_types + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + self.content_kinds + } + + fn config_schema(&self) -> crate::config::ConfigSchema { + self.schema.clone() + } +} + +/// Run each plugin's `configure()` against its per-plugin cache dir and return +/// the plugins keyed by `upstream_id`. Never errors on a per-plugin problem: +/// `MissingConfig` is soft-skipped, every other failure becomes a +/// [`DegradedPlugin`] (see [`serve_feeders`]). Public so feeder binaries / +/// tests can configure without serving. +pub fn configure_plugins( + plugins: Vec>, + state_dir: &FsPath, +) -> anyhow::Result>> { + let mut out: HashMap> = HashMap::new(); + for mut plugin in plugins { + let id = plugin.upstream_id(); + // Captured up front (both are `'static`, so this borrow of `plugin` + // ends immediately) — lets us build a DegradedPlugin without holding a + // borrow across `configure(&mut self)` / the move into `out`. + let file_types = plugin.served_file_types(); + let content_kinds = plugin.served_content_kinds(); + // Preserve the config surface so a degraded stand-in still serves the + // real config form (see DegradedPlugin.schema). + let schema = plugin.config_schema(); + let cache_dir = state_dir.join("gateway").join(id); + + // Build (don't insert) a degraded stand-in: keeps the container up and + // surfaces the reason via /health instead of crash-looping. + let make_degraded = |reason: String| -> Arc { + warn!( + target: "meta-feeder", + upstream_id = id, + %reason, + "feeder plugin degraded: container stays up; error surfaced via /health" + ); + Arc::new(DegradedPlugin { + id, + reason, + file_types, + content_kinds, + schema: schema.clone(), + }) + }; + + if let Err(e) = std::fs::create_dir_all(&cache_dir) { + let reason = format!("cache dir {} setup failed: {e}", cache_dir.display()); + out.insert(id.to_string(), make_degraded(reason)); + continue; + } + + match plugin.configure(&cache_dir) { + Ok(()) => { + out.insert(id.to_string(), Arc::from(plugin)); + } + Err(ConfigError::MissingConfig { plugin: p, what }) => { + // Keep a degraded stand-in (don't drop it) so its config form + // stays reachable from the dashboard and the operator can supply + // the missing config — then restart the feeder to apply it. + warn!( + target: "meta-feeder", + upstream_id = %p, + missing = %what, + "feeder plugin degraded: required config not supplied; \ + config form still served so it can be fixed from the dashboard" + ); + out.insert( + id.to_string(), + make_degraded(format!("required config not supplied: {what}")), + ); + } + Err(other) => { + out.insert(id.to_string(), make_degraded(other.to_string())); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::DiscoveryRecord; + use async_trait::async_trait; + use futures::stream::BoxStream; + use std::collections::BTreeMap; + + /// A feeder plugin that overrides `handle_query_stream` to emit a Base, then + /// an EnrichPatch, then Done — so the NDJSON route is exercised on a real + /// incremental stream (not just the collect-default). + struct StubStreaming; + + #[async_trait] + impl FeederPlugin for StubStreaming { + fn upstream_id(&self) -> &'static str { + "stub" + } + fn configure(&mut self, _cache_dir: &FsPath) -> Result<(), ConfigError> { + Ok(()) + } + async fn handle_query( + &self, + _q: &GatewayQuery, + _n: usize, + ) -> Result, GatewayError> { + Ok(vec![]) + } + async fn handle_query_stream( + &self, + _q: &GatewayQuery, + _n: usize, + ) -> Result, GatewayError> { + let rec = DiscoveryRecord { + upstream_id: "stub".into(), + record_id: "1".into(), + fields: BTreeMap::new(), + }; + let events = vec![ + GatewaySearchEvent::Base(rec), + GatewaySearchEvent::EnrichPatch { + record_id: "1".into(), + set: BTreeMap::from([("title".to_string(), "X".to_string())]), + remove: vec![], + }, + GatewaySearchEvent::Done, + ]; + Ok(Box::pin(futures::stream::iter(events))) + } + async fn compute_outcomes( + &self, + _record_id: &str, + ) -> Result, GatewayError> { + Ok(vec![]) + } + } + + #[tokio::test] + async fn query_stream_emits_ndjson_events_in_order() { + let mut plugins: HashMap> = HashMap::new(); + plugins.insert("stub".to_string(), Arc::new(StubStreaming)); + let app = router(plugins, "test".to_string(), std::env::temp_dir()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let body = reqwest::Client::new() + .post(format!("http://{addr}/query_stream")) + .json(&serde_json::json!({ + "upstream_id": "stub", + "query": {"raw_text":"x","free_text":"x","filters":{},"ranges":[],"negations":[]}, + "max_results": 5 + })) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + let events: Vec = body + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).expect("each NDJSON line is a GatewaySearchEvent")) + .collect(); + + assert_eq!(events.len(), 3, "body was: {body:?}"); + assert!(matches!(events[0], GatewaySearchEvent::Base(_))); + assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. })); + assert!(matches!(events[2], GatewaySearchEvent::Done)); + } +} diff --git a/crates/meta-feeder-sdk/src/types.rs b/crates/meta-feeder-sdk/src/types.rs new file mode 100644 index 0000000..9fbc7b6 --- /dev/null +++ b/crates/meta-feeder-sdk/src/types.rs @@ -0,0 +1,124 @@ +//! In-process types used by the gateway runtime, feeders, and plugins. +//! +//! `GatewayError` here is **distinct from** the wire-stable +//! `GatewayWireError` (see [`crate::query`]): this one can carry rich +//! internal context (anyhow chains, plugin specifics); the wire one is narrow +//! on purpose since adding variants is a protocol change. Conversion happens +//! at the protocol boundary. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// Content-addressed hash string carried over the wire. Renamed from +/// `Midhash` in the multi-outcome plugin contract migration — the name +/// `Midhash` lied for sha2-256 CIDs (which gateway plugins also emit +/// via `compute_ipfs_cid`). The carrier is generic; the hash family is +/// discriminated by `HashKind` on `HashOutcome`. +/// +/// Shape: base32-lower 'b'-prefixed multibase CIDv1. The specific +/// multihash inside (`0x1000` for midhash256, `0x12` for sha2-256) is +/// determined by the producing plugin's hashing path. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Hash(pub String); + +impl Hash { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for Hash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Pre-hash identifier for a record discovered through a gateway. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct DiscoveryId { + pub upstream_id: String, + pub record_id: String, +} + +/// Metadata-only search result. Distinct from meta-core's persistent record +/// (which is a flat `map[string]string` indexed by midhash); a +/// `DiscoveryRecord` only lives inside the gateway search/response pipeline. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DiscoveryRecord { + pub upstream_id: String, + pub record_id: String, + pub fields: BTreeMap, +} + +/// In-process error returned by `FeederPlugin` methods. +#[derive(Debug, thiserror::Error)] +pub enum GatewayError { + #[error("upstream transient error: {0}")] + Transient(String), + + #[error("upstream permanent error: {0}")] + Permanent(String), + + #[error("rate limited; retry in {retry_after_s}s")] + RateLimited { retry_after_s: u32 }, + + #[error("record not found upstream")] + NotFound, + + #[error("internal: {0}")] + Internal(#[from] anyhow::Error), +} + +/// Byte stream returned by a plugin's `handle_fetch`. +pub type ByteStream = futures::stream::BoxStream<'static, Result>; + +/// Plugin liveness snapshot surfaced via `/health`. Must not call upstream +/// — `health()` is invoked on every status poll. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "state")] +pub enum PluginHealth { + Ok, + Degraded { reason: String }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hash_display_and_as_str_match() { + let m = Hash("bafy123".to_string()); + assert_eq!(m.as_str(), "bafy123"); + assert_eq!(format!("{m}"), "bafy123"); + } + + #[test] + fn discovery_record_roundtrips_through_serde_json() { + let r = DiscoveryRecord { + upstream_id: "scihub".to_string(), + record_id: "10.1038/s41586-021-03819-2".to_string(), + fields: BTreeMap::from_iter([ + ("title".to_string(), "AlphaFold".to_string()), + ("year".to_string(), "2021".to_string()), + ]), + }; + let j = serde_json::to_string(&r).expect("serialize"); + let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize"); + assert_eq!(r, back); + } + + #[test] + fn plugin_health_serialises_with_tag() { + let ok = serde_json::to_value(PluginHealth::Ok).unwrap(); + assert_eq!(ok, serde_json::json!({"state": "ok"})); + let bad = serde_json::to_value(PluginHealth::Degraded { + reason: "upstream 5xx".into(), + }) + .unwrap(); + assert_eq!( + bad, + serde_json::json!({"state": "degraded", "reason": "upstream 5xx"}) + ); + } +} diff --git a/feeder-plugin/usenet-feeder/Cargo.toml b/feeder-plugin/usenet-feeder/Cargo.toml new file mode 100644 index 0000000..a0bbbb6 --- /dev/null +++ b/feeder-plugin/usenet-feeder/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "usenet-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Usenet feeder sidecar: bridges a Usenet header scan we run OURSELVES into the +# MetaMesh network. Unlike `meta-feeder-indexer` (which proxies an external +# Newznab/torznab indexer) this feeder owns the metadata half — it reads an +# nntmux sidecar's release catalog and mints a PORTABLE `nzb-posting` (0x1003) +# cid from the article Message-IDs, so any peer with a plain NNTP provider can +# redeem it without an indexer credential. +# +# It ADDS a Usenet metadata source alongside meta-feeder-indexer; it does not +# replace it. Like every feeder it is meta-core-free + blockstore-free (the +# gateway core owns hashing/seeding/store-back — see the gateway's invariant 10). +# +# See `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md`. + +[lib] +name = "usenet_feeder" +path = "src/lib.rs" + +[[bin]] +name = "usenet-feeder" +path = "src/main.rs" + +[dependencies] +meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +thiserror = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +bytes = "1" +futures = "0.3" +regex = "1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } + +# nntmux's release catalog lives in MariaDB. `sqlx` in offline-free runtime mode +# (no compile-time macro checking) so the build needs no live DB. +sqlx = { version = "0.8", default-features = false, features = [ + "runtime-tokio-rustls", + "mysql", + "macros", + "chrono", +] } +# nntmux stores each release's manifest as a gzipped NZB on disk; the ordered +# Message-IDs survive ONLY there (it purges collections/binaries/parts after +# writing the file), so reading + gunzipping it is the load-bearing path. +flate2 = "1" +quick-xml = { version = "0.36", features = ["serialize"] } + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/usenet-feeder/Dockerfile b/feeder-plugin/usenet-feeder/Dockerfile new file mode 100644 index 0000000..749650f --- /dev/null +++ b/feeder-plugin/usenet-feeder/Dockerfile @@ -0,0 +1,57 @@ +# syntax=docker/dockerfile:1 +# ============================================================================= +# usenet-feeder — Usenet feeder sidecar over an nntmux header-scan catalog. +# Multi-stage build. +# +# Build context must be this repo root (packages/plugins/meta-feeder-usenet) so +# the `meta-feeder-sdk` path dependency resolves. Same BuildKit cache-mount +# strategy as the other feeders. +# +# This feeder talks to MariaDB (sqlx, rustls — no system MySQL client needed) +# and reads nntmux's NZB store off a shared volume. It holds no media bytes. +# ============================================================================= + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + ca-certificates \ + git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY crates/ crates/ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=meta-feeder-usenet-target,target=/build/target \ + cargo build --release --bin usenet-feeder \ + && cp target/release/usenet-feeder /usenet-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usenet-feeder /usr/local/bin/usenet-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/usenet-feeder"] diff --git a/feeder-plugin/usenet-feeder/src/lib.rs b/feeder-plugin/usenet-feeder/src/lib.rs new file mode 100644 index 0000000..a1d0bce --- /dev/null +++ b/feeder-plugin/usenet-feeder/src/lib.rs @@ -0,0 +1,26 @@ +//! `usenet-feeder` library surface. The binary (`main.rs`) wraps the plugin in +//! [`meta_feeder_sdk::serve_feeders`]. +//! +//! This feeder owns the **metadata half** of Usenet: it reads an nntmux header- +//! scan catalog we run ourselves and mints a portable `nzb-posting` (`0x1003`) +//! cid from each release's article Message-IDs. Unlike `meta-feeder-indexer`'s +//! `0x1005` locator, that cid embeds no indexer host, so any peer with a plain +//! NNTP provider can redeem it — no indexer credential anywhere in the path. +//! +//! It ADDS a source alongside meta-feeder-indexer; it replaces nothing. +//! +//! Like every feeder it is meta-core-free and blockstore-free: it finds records +//! and serves bytes, and the gateway core owns hashing-into-the-blockstore, the +//! meta-core store-back, and the libp2p wire (gateway invariant 10). +//! +//! See `meta-gateway/docs/others/self-hosted-usenet-indexer-study.md`. + +/// Read-only access to the nntmux sidecar: its MariaDB release catalog and the +/// on-disk gzip NZBs that are the *only* durable home of the Message-IDs. +pub mod nntmux { + pub mod db; + pub mod nzb; +} + +/// The `usenet` upstream itself. +pub mod usenet; diff --git a/feeder-plugin/usenet-feeder/src/main.rs b/feeder-plugin/usenet-feeder/src/main.rs new file mode 100644 index 0000000..f856221 --- /dev/null +++ b/feeder-plugin/usenet-feeder/src/main.rs @@ -0,0 +1,43 @@ +//! `usenet-feeder` — single-source Usenet feeder sidecar over an nntmux +//! header-scan catalog. +//! +//! One plugin (`usenet`) served through [`meta_feeder_sdk::serve_feeders`]. +//! +//! Operator config — the nntmux database URL and NZB store path — is read from +//! the persisted `config.json` (dashboard-written), with the env vars below as a +//! **first-boot seed only**: the file wins on the next restart, and there is no +//! hot reload (gateway invariant 12). +//! +//! Env (seed): `NNTMUX_DB_URL`, `NNTMUX_NZB_PATH`. +//! Env (infra): `META_FEEDER_HTTP_LISTEN` (default `0.0.0.0:8080`), +//! `META_FEEDER_STATE_DIR` (default `/data/meta-feeder`), `RUST_LOG`. + +use std::net::SocketAddr; + +use meta_feeder_sdk::plugin::FeederPlugin; +use meta_feeder_sdk::serve_feeders; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use usenet_feeder::usenet::UsenetPlugin; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(UsenetPlugin::new())]; + + info!(target: "meta-feeder", loaded = plugins.len(), "usenet feeder starting"); + + serve_feeders(plugins, state_dir, listen).await +} diff --git a/feeder-plugin/usenet-feeder/src/nntmux/db.rs b/feeder-plugin/usenet-feeder/src/nntmux/db.rs new file mode 100644 index 0000000..d8b39db --- /dev/null +++ b/feeder-plugin/usenet-feeder/src/nntmux/db.rs @@ -0,0 +1,281 @@ +//! Read-only access to the nntmux sidecar's release catalog (MariaDB). +//! +//! We are a **consumer** of nntmux's schema, never a writer: the scanner owns +//! that database, and everything in it is re-derivable by re-scanning. Treat it +//! as persistent cache, not as a source of truth (the Tribler precedent — a +//! stateful sidecar the gateway still sees as a stateless HTTP feeder). +//! +//! Only a narrow, long-stable column set is selected (`guid`, `searchname`, +//! `name`, `size`, `postdate`, `categories_id`, `totalpart`). nntmux carries +//! ~80 columns on `releases`, most of them scanner bookkeeping; binding a wide +//! row here would make us fragile to upstream schema churn for no gain. + +use anyhow::{Context, Result}; +use sqlx::mysql::{MySqlPool, MySqlPoolOptions}; +use sqlx::Row; + +/// One collated nntmux release — the metadata half of a Usenet posting. +#[derive(Debug, Clone)] +pub struct Release { + /// nntmux's public id, and the basename of the on-disk `.nzb.gz`. + /// This is our `record_id`. + pub guid: String, + /// The collator's cleaned, scene-style name + /// (`Celebrity.Ghost.Stories.S05E08.720p.HDTV.x264-DHD`). Falls back to the + /// raw posted `name` when the collator could not improve on it. + pub search_name: String, + /// Total posted size in bytes (RAR + par2 included, so an over-estimate of + /// the final media file). + pub size: u64, + /// nntmux category id (`categories.id`). Coarse: 2000s = movies, 5000s = TV, + /// 3000s = audio, 7000s = books — the Newznab category tree. + pub category_id: i32, + /// Unix seconds of the original posting date, when nntmux recorded one. + pub post_date: Option, +} + +/// Connection pool to nntmux's MariaDB. Small by design — this feeder issues a +/// handful of short SELECTs per query, and the scanner needs the headroom. +#[derive(Clone)] +pub struct Db { + pool: MySqlPool, +} + +impl Db { + /// Open a pool. Does **not** verify the schema — a feeder must come up and + /// serve `/health` even when its upstream is unreachable, so the gateway's + /// `depends_on` is satisfied and discovery does not soft-skip us forever. + pub async fn connect(url: &str) -> Result { + let pool = MySqlPoolOptions::new() + .max_connections(4) + .acquire_timeout(std::time::Duration::from_secs(10)) + .connect(url) + .await + .with_context(|| "connect to nntmux mariadb")?; + Ok(Self { pool }) + } + + /// Free-text search over the collated release names. + /// + /// `LIKE %term%` on `searchname` — deliberately simple. nntmux can be built + /// with Manticore/Elasticsearch for real full-text search, but both are + /// optional in the minimum stack and we do not want to require them. If + /// search quality becomes the bottleneck, that is the upgrade path. + pub async fn search(&self, term: &str, limit: usize) -> Result> { + // ⚠ TOKENISE, don't match the phrase. Scene releases are dot-separated + // (`Breaking.Bad.S01E01.1080p...`) while users type spaces ("breaking + // bad"), so a single `LIKE %breaking bad%` matches essentially NOTHING + // — the separator never lines up. Split into words and AND them, which + // makes the separator irrelevant on both sides. + // + // Found by end-to-end search from meta-watch; the unit tests missed it + // because they only exercised the escaping, never a realistic + // scene-style name against a natural-language query. + let words = search_tokens(term); + if words.is_empty() { + return Ok(vec![]); + } + let mut sql = String::from( + "SELECT guid, searchname, name, size, categories_id, \ + UNIX_TIMESTAMP(postdate) AS postdate_unix \ + FROM releases WHERE ", + ); + sql.push_str( + &words + .iter() + .map(|_| "searchname LIKE ?") + .collect::>() + .join(" AND "), + ); + sql.push_str(" ORDER BY postdate DESC LIMIT ?"); + + let mut q = sqlx::query(&sql); + for w in &words { + q = q.bind(format!("%{}%", escape_like(w))); + } + let rows = q + .bind(limit as i64) + .fetch_all(&self.pool) + .await + .context("query releases by name")?; + Ok(rows.iter().map(row_to_release).collect()) + } + + /// Look one release up by guid — the `compute_outcomes` path, where the + /// record_id came back to us from the gateway. + pub async fn by_guid(&self, guid: &str) -> Result> { + let row = sqlx::query( + "SELECT guid, searchname, name, size, categories_id, \ + UNIX_TIMESTAMP(postdate) AS postdate_unix \ + FROM releases WHERE guid = ? LIMIT 1", + ) + .bind(guid) + .fetch_optional(&self.pool) + .await + .context("query release by guid")?; + Ok(row.as_ref().map(row_to_release)) + } + + /// Cheap liveness probe for `health()` — confirms the catalog is reachable + /// *and* non-empty. An empty `releases` table is the visible symptom of both + /// nntmux silent-zero traps (an unset `PATH_TO_NZBS`, or groups left + /// `active=0`), so reporting it as degraded turns a silent failure into a + /// legible one. + pub async fn release_count(&self) -> Result { + let row = sqlx::query("SELECT COUNT(*) AS n FROM releases") + .fetch_one(&self.pool) + .await + .context("count releases")?; + Ok(row.try_get::("n").unwrap_or(0)) + } +} + +fn row_to_release(row: &sqlx::mysql::MySqlRow) -> Release { + let search_name: String = row + .try_get::("searchname") + .ok() + .filter(|s| !s.is_empty()) + .or_else(|| row.try_get::("name").ok()) + .unwrap_or_default(); + Release { + guid: row.try_get::("guid").unwrap_or_default(), + search_name, + // nntmux stores size as an unsigned bigint; it arrives as u64 or + // (older schemas) a decimal string. + size: row + .try_get::("size") + .ok() + .or_else(|| { + row.try_get::("size") + .ok() + .and_then(|s| s.parse().ok()) + }) + .unwrap_or(0), + category_id: row.try_get::("categories_id").unwrap_or(0), + post_date: row.try_get::("postdate_unix").ok(), + } +} + +/// Split a user query into match tokens. +/// +/// Anything that is not alphanumeric is a separator — which folds the scene +/// world's `.`/`_`/`-` and the human world's space onto the same footing, so +/// "breaking bad", "Breaking.Bad" and "breaking-bad" all produce the same +/// tokens and all match `Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE`. +/// +/// Single characters are dropped: they match nearly every row and cost a full +/// scan for no selectivity. +fn search_tokens(term: &str) -> Vec { + term.split(|c: char| !c.is_alphanumeric()) + .filter(|w| w.chars().count() > 1) + .map(|w| w.to_string()) + .collect() +} + +/// Escape the `LIKE` wildcards so a user searching for `100%` does not match +/// everything. Backslash is MySQL's default escape character. +fn escape_like(s: &str) -> String { + s.replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_") +} + +/// Map an nntmux/Newznab category id onto MetaMesh's `fileType`. +/// +/// The Newznab tree is coarse and stable: 1000s console, 2000s movies, 3000s +/// audio, 4000s PC, 5000s TV, 6000s XXX, 7000s books. We only need the buckets +/// `METADATA_KEYS.md` §1 defines. +pub fn file_type_for_category(category_id: i32) -> &'static str { + match category_id / 1000 { + 2 | 5 | 6 => "video", + 3 => "audio", + 7 => "document", + _ => "archive", + } +} + +/// `contentKind` refinement of [`file_type_for_category`]. Only asserted where +/// the category tree is unambiguous — a wrong `contentKind` is worse than none, +/// because meta-watch's curated rows gate on it. +pub fn content_kind_for_category(category_id: i32) -> Option<&'static str> { + match category_id / 1000 { + 2 => Some("movie"), + 5 => Some("tv"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// ⚠ THE DOT-VS-SPACE REGRESSION. Scene releases are dot-separated and + /// users type spaces; matching the raw phrase finds nothing. Caught by an + /// end-to-end search from meta-watch that returned 0 results against a + /// catalog that plainly contained the show. + #[test] + fn a_spaced_query_tokenises_to_match_a_dotted_scene_name() { + let tokens = search_tokens("breaking bad"); + assert_eq!(tokens, vec!["breaking", "bad"]); + + // Every token must be a substring of the real scene name — that is + // exactly what the AND-ed LIKEs check. + let scene = "Breaking.Bad.S01E01.1080p.BluRay.x264-SCENE".to_lowercase(); + assert!( + tokens.iter().all(|t| scene.contains(&t.to_lowercase())), + "tokens {tokens:?} must all appear in {scene}" + ); + + // The naive whole-phrase form does NOT — this is the bug, pinned. + assert!( + !scene.contains("breaking bad"), + "if this ever passes, the dotted-name assumption changed" + ); + } + + /// Separators are interchangeable in both directions. + #[test] + fn scene_and_human_separators_produce_the_same_tokens() { + let expect = vec!["breaking".to_string(), "bad".to_string()]; + assert_eq!(search_tokens("breaking bad"), expect); + assert_eq!(search_tokens("Breaking.Bad").iter().map(|s| s.to_lowercase()).collect::>(), expect); + assert_eq!(search_tokens("breaking-bad"), expect); + assert_eq!(search_tokens("breaking_bad"), expect); + assert_eq!(search_tokens(" breaking bad "), expect); + } + + /// Single characters are dropped (no selectivity, full scan) and an + /// all-noise query yields nothing rather than matching the whole catalog. + #[test] + fn noise_only_queries_yield_no_tokens() { + assert!(search_tokens("").is_empty()); + assert!(search_tokens(" ").is_empty()); + assert!(search_tokens("- . _").is_empty()); + assert_eq!(search_tokens("a bc"), vec!["bc"]); + } + + #[test] + fn like_wildcards_are_escaped() { + assert_eq!(escape_like("100%"), "100\\%"); + assert_eq!(escape_like("a_b"), "a\\_b"); + assert_eq!(escape_like("c:\\d"), "c:\\\\d"); + assert_eq!(escape_like("naruto"), "naruto"); + } + + #[test] + fn newznab_categories_map_to_file_types() { + assert_eq!(file_type_for_category(2040), "video"); // movies HD + assert_eq!(file_type_for_category(5040), "video"); // tv HD + assert_eq!(file_type_for_category(3010), "audio"); + assert_eq!(file_type_for_category(7020), "document"); // ebook + assert_eq!(file_type_for_category(4000), "archive"); // pc + } + + #[test] + fn content_kind_only_where_unambiguous() { + assert_eq!(content_kind_for_category(2040), Some("movie")); + assert_eq!(content_kind_for_category(5040), Some("tv")); + assert_eq!(content_kind_for_category(3010), None); + assert_eq!(content_kind_for_category(7020), None); + } +} diff --git a/feeder-plugin/usenet-feeder/src/nntmux/nzb.rs b/feeder-plugin/usenet-feeder/src/nntmux/nzb.rs new file mode 100644 index 0000000..d0718ca --- /dev/null +++ b/feeder-plugin/usenet-feeder/src/nntmux/nzb.rs @@ -0,0 +1,187 @@ +//! Reading an nntmux release's manifest off disk and extracting its ordered +//! article Message-IDs. +//! +//! # Why the file and not the database +//! +//! You **cannot** read a finished release's Message-IDs out of nntmux's +//! database. `NzbService::writeNzbForReleaseId()` writes the release's NZB and +//! then *unconditionally* purges collections → binaries → parts (all FKs are +//! `ON DELETE CASCADE`, and there is no retention setting). Those tables are +//! transient staging for the header scan; for any *collated* release the +//! ordered Message-IDs survive **only** in the on-disk gzip NZB at +//! `//.nzb.gz`. +//! +//! This was confirmed on the live path during the spike (the offline +//! `import-nzbs` path never stages collections/binaries/parts at all, so it +//! cannot be used to check this). It is the single load-bearing fact behind the +//! whole feeder: lose it and you are back to grabbing `.nzb` files from an +//! external indexer, which is exactly what this design exists to avoid. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use flate2::read::GzDecoder; + +/// A parsed nntmux manifest: the raw NZB XML plus the Message-IDs in posting +/// order. +#[derive(Debug, Clone)] +pub struct Manifest { + /// The decompressed NZB XML. Handed to the gateway verbatim so it can seed + /// it as an ordinary sha2-256 cid — the consumer parses this exact document. + pub xml: Vec, + /// Every `` Message-ID across every ``, in document order. + /// Bare form (no surrounding `<>`), matching how NZB stores them and how + /// meta-share's `NzbSegment.message_id` holds them. + pub message_ids: Vec, +} + +/// nntmux shards its NZB storage one level deep by the guid's **first +/// character** (`storage/nzb/a/abcdef….nzb.gz`). Mirrored here rather than +/// globbed: a scan of a large store is slow and this is on the compute path. +pub fn nzb_path(nzb_root: &Path, guid: &str) -> Option { + let first = guid.chars().next()?; + Some(nzb_root.join(first.to_string()).join(format!("{guid}.nzb.gz"))) +} + +/// Read + gunzip a release's `.nzb.gz` and pull out its Message-IDs. +/// +/// Returns `Ok(None)` when the file is absent — a normal, non-fatal state: a +/// release row can exist before `createNZBs()` has written its manifest, and +/// the caller simply skips that record rather than failing the whole query. +pub fn load(nzb_root: &Path, guid: &str) -> Result> { + let Some(path) = nzb_path(nzb_root, guid) else { + return Ok(None); + }; + if !path.exists() { + return Ok(None); + } + let raw = std::fs::read(&path).with_context(|| format!("read nzb {}", path.display()))?; + let mut xml = Vec::new(); + GzDecoder::new(&raw[..]) + .read_to_end(&mut xml) + .with_context(|| format!("gunzip nzb {}", path.display()))?; + + let message_ids = extract_message_ids(&xml) + .with_context(|| format!("parse nzb {}", path.display()))?; + if message_ids.is_empty() { + anyhow::bail!("nzb {} has no segments", path.display()); + } + Ok(Some(Manifest { xml, message_ids })) +} + +/// Pull every `` body out of an NZB document, in order. +/// +/// Deliberately a light scan rather than a full deserialize: we need exactly +/// one thing from this document, the consumer (meta-share) does its own real +/// parse of the same bytes, and a strict schema here would reject NZBs that +/// meta-share would happily play. Surrounding `<>` are stripped if a producer +/// included them — the mint normalises this too, but keeping the stored form +/// canonical means the two never disagree about what was hashed. +fn extract_message_ids(xml: &[u8]) -> Result> { + use quick_xml::events::Event; + use quick_xml::Reader; + + let text = std::str::from_utf8(xml).context("nzb is not utf-8")?; + let mut reader = Reader::from_str(text); + reader.config_mut().trim_text(true); + + let mut ids = Vec::new(); + let mut in_segment = false; + loop { + match reader.read_event() { + Ok(Event::Start(e)) if e.local_name().as_ref() == b"segment" => in_segment = true, + Ok(Event::End(e)) if e.local_name().as_ref() == b"segment" => in_segment = false, + Ok(Event::Text(t)) if in_segment => { + let raw = t.unescape().context("unescape segment text")?; + let id = raw + .trim() + .trim_start_matches('<') + .trim_end_matches('>') + .to_string(); + if !id.is_empty() { + ids.push(id); + } + } + Ok(Event::Eof) => break, + Err(e) => return Err(anyhow::anyhow!("nzb xml: {e}")), + _ => {} + } + } + Ok(ids) +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE: &str = r#" + + + alt.binaries.teevee + + aaa@astraweb.com + bbb@astraweb.com + + + + + <ccc@astraweb.com> + + +"#; + + #[test] + fn extracts_ids_in_document_order() { + let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse"); + assert_eq!( + ids, + vec![ + "aaa@astraweb.com".to_string(), + "bbb@astraweb.com".to_string(), + "ccc@astraweb.com".to_string(), + ] + ); + } + + /// Angle brackets are stripped so the stored form matches meta-share's + /// `NzbSegment.message_id` (bare) and the mint's normalisation. + #[test] + fn strips_angle_brackets() { + let ids = extract_message_ids(SAMPLE.as_bytes()).expect("parse"); + assert!(ids.iter().all(|i| !i.contains('<') && !i.contains('>'))); + } + + #[test] + fn nzb_path_shards_on_first_guid_char() { + let p = nzb_path(Path::new("/s/nzb"), "abc123").expect("path"); + assert_eq!(p, Path::new("/s/nzb/a/abc123.nzb.gz")); + } + + /// A release row can exist before `createNZBs()` has written the manifest. + /// That is normal — the caller skips the record, it is not an error. + #[test] + fn missing_file_is_none_not_error() { + let dir = tempfile::tempdir().expect("tmp"); + let got = load(dir.path(), "deadbeef").expect("no error"); + assert!(got.is_none()); + } + + #[test] + fn round_trips_a_gzipped_nzb() { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + + let dir = tempfile::tempdir().expect("tmp"); + std::fs::create_dir_all(dir.path().join("a")).expect("mkdir"); + let mut enc = GzEncoder::new(Vec::new(), Compression::default()); + enc.write_all(SAMPLE.as_bytes()).expect("gz write"); + let gz = enc.finish().expect("gz finish"); + std::fs::write(dir.path().join("a/abc.nzb.gz"), gz).expect("write"); + + let m = load(dir.path(), "abc").expect("load").expect("present"); + assert_eq!(m.message_ids.len(), 3); + assert_eq!(m.xml, SAMPLE.as_bytes()); + } +} diff --git a/feeder-plugin/usenet-feeder/src/usenet.rs b/feeder-plugin/usenet-feeder/src/usenet.rs new file mode 100644 index 0000000..40c3227 --- /dev/null +++ b/feeder-plugin/usenet-feeder/src/usenet.rs @@ -0,0 +1,396 @@ +//! The `usenet` upstream — a `FeederPlugin` over an nntmux header-scan catalog. +//! +//! # What makes this different from `meta-feeder-indexer` +//! +//! indexer-feeder proxies an **external** Newznab indexer and mints an +//! `nzb-release` (`0x1005`) cid that *embeds that indexer's host*. Only a peer +//! holding that indexer's credential can ever redeem it. +//! +//! This feeder scans Usenet itself, so it already holds the article Message-IDs +//! at collation time — no grab, no scrape. That lets it mint a **portable** +//! `nzb-posting` (`0x1003`) cid: a digest over the Message-ID set, embedding no +//! host, redeemable by any peer with a plain NNTP provider. +//! +//! The two coexist. This ADDS a Usenet metadata source; it removes nothing. +//! +//! # The manifest hand-off (the one piece of new machinery) +//! +//! A `0x1003` cid is a *hash*, not a locator: it is not reversible to the +//! Message-IDs, and it can never be bitswap-fetched under itself (bitswap +//! derives a block's cid by hashing the block). So the `.nzb` must travel +//! separately. We emit `manifest_url` — a **relative** URL onto our own +//! `/blob/...` route — and the gateway core fetches it, seeds it as an ordinary +//! sha2-256 IPFS cid, publishes it as a first-class `fileType=nzb` meta-core +//! record, and rewrites the field to that cid. Identical to how posters are +//! handled. See the study's §5.3. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use meta_feeder_sdk::config::{ConfigField as F, ConfigSchema}; +use meta_feeder_sdk::hash::compute_nzb_posting_cid; +use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; +use meta_feeder_sdk::query::GatewayQuery; +use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth}; +use tokio::sync::RwLock; +use tracing::{debug, info, warn}; + +use crate::nntmux::db::{content_kind_for_category, file_type_for_category, Db, Release}; +use crate::nntmux::nzb; + +pub const UPSTREAM_ID: &str = "usenet"; + +/// Operator-supplied settings, persisted to `/config.json` by the +/// SDK's config surface. Env is a **seed only** — the file wins on the next +/// restart (invariant 12; there is no hot reload). +#[derive(Debug, Clone, Default)] +pub struct Settings { + /// `mysql://user:pass@host:3306/nntmux` + pub db_url: String, + /// Root of nntmux's NZB store, shared with the sidecar as a volume. The + /// directory nntmux's `PATH_TO_NZBS` points at. + pub nzb_root: String, +} + +impl Settings { + fn from_env() -> Self { + Self { + db_url: std::env::var("NNTMUX_DB_URL").unwrap_or_default(), + nzb_root: std::env::var("NNTMUX_NZB_PATH") + .unwrap_or_else(|_| "/nntmux-nzb".to_string()), + } + } + + fn merge_file(&mut self, cache_dir: &Path) { + let path = cache_dir.join("config.json"); + let Ok(raw) = std::fs::read_to_string(&path) else { + return; + }; + let Ok(v) = serde_json::from_str::(&raw) else { + warn!(target: "meta-feeder", path = %path.display(), "config.json is not valid json; using env seed"); + return; + }; + if let Some(s) = v.get("db_url").and_then(|x| x.as_str()) { + if !s.is_empty() { + self.db_url = s.to_string(); + } + } + if let Some(s) = v.get("nzb_root").and_then(|x| x.as_str()) { + if !s.is_empty() { + self.nzb_root = s.to_string(); + } + } + } +} + +pub struct UsenetPlugin { + settings: Settings, + /// `None` until a successful connect. A feeder with unreachable config + /// **must still serve `/health`** so the gateway's `depends_on` is satisfied + /// — it soft-skips its upstream instead of failing to boot (invariant 10). + db: Arc>>, + nzb_root: PathBuf, +} + +impl Default for UsenetPlugin { + fn default() -> Self { + Self::new() + } +} + +impl UsenetPlugin { + pub fn new() -> Self { + Self { + settings: Settings::default(), + db: Arc::new(RwLock::new(None)), + nzb_root: PathBuf::new(), + } + } + + async fn db(&self) -> Result { + self.db + .read() + .await + .clone() + .ok_or_else(|| GatewayError::Permanent("nntmux database not configured".into())) + } + + /// Mint a release's `nzb-posting` cid by reading its manifest off disk. + /// + /// `None` when the `.nzb.gz` is absent (a release row can exist before + /// `createNZBs()` has written it) or unreadable — the caller then emits a + /// record without a cid, which is honest: we genuinely cannot address that + /// posting yet. + fn posting_cid(&self, guid: &str) -> Option { + match nzb::load(&self.nzb_root, guid) { + Ok(Some(m)) => Some(compute_nzb_posting_cid(&m.message_ids)), + Ok(None) => None, + Err(e) => { + warn!(target: "meta-feeder", %guid, error = %e, "usenet: manifest unreadable; record will carry no cid"); + None + } + } + } + + /// Build the discovery record for one release. Metadata only — the bytes + /// live on Usenet and are fetched by meta-share at playback, never by us. + /// + /// ⚠ **`cid` must be `Some` on the SEARCH path, not just on compute.** A + /// record whose fields carry no `cids/` member reaches the client as a + /// bare `gateway::` reference, which meta-share + /// refuses to parse (the `:` token form was removed) — so the + /// title renders but is unplayable. `nzb-release` (`0x1005`) avoids this + /// for free because its cid is derivable from `{host, id}` alone; ours is a + /// digest over the Message-ID set, so it has to be minted from the manifest + /// — which is a **local file read**, no network and no indexer grab, and + /// therefore fine to do per search hit. Caught by an end-to-end play from + /// meta-watch that landed on "Unavailable". + fn record_for(&self, r: &Release, cid: Option<&str>) -> DiscoveryRecord { + let mut fields: BTreeMap = BTreeMap::new(); + // The bare-cid key-set (`cids/` = "true") — the ONLY shape + // meta-core indexes and the one meta-search elects the canonical cid + // from (METADATA_KEYS §2). A flat `cid` field would be inert. + if let Some(c) = cid { + fields.insert(format!("cids/{c}"), "true".to_string()); + // ⚠ EMITTED ON THE SEARCH PATH, NOT JUST ON COMPUTE. The gateway + // core rewrites this to a content-addressed `manifest` cid (its + // SEEDABLE_FIELDS table) and persists the record — and it does that + // for *search* hits, because the search path is the one that + // actually runs: `GatewayOp::Hash` (compute) is not sent by any + // client in the fleet today, so anything emitted only from + // `compute_outcomes` is never seen. Without this on search, the + // stored record has no `manifest`, and meta-share cannot resolve + // the posting at playback ("Unavailable" in meta-watch). + // + // Relative on purpose: we do not know our own externally-reachable + // address; the core resolves it against the URL it discovered us + // on (`resolve_seed_url`). + fields.insert( + "manifest_url".to_string(), + format!("/blob/{UPSTREAM_ID}/{}.nzb", r.guid), + ); + } + // Canonical per-upstream id field (`id`) — required of + // every record by the SDK's field-naming convention. + fields.insert(format!("{UPSTREAM_ID}id"), r.guid.clone()); + fields.insert("title".to_string(), r.search_name.clone()); + fields.insert("fileName".to_string(), r.search_name.clone()); + if r.size > 0 { + fields.insert("sizeByte".to_string(), r.size.to_string()); + } + fields.insert( + "fileType".to_string(), + file_type_for_category(r.category_id).to_string(), + ); + if let Some(kind) = content_kind_for_category(r.category_id) { + fields.insert("contentKind".to_string(), kind.to_string()); + } + if let Some(ts) = r.post_date { + fields.insert("publishedAt".to_string(), ts.to_string()); + } + // Provenance: this catalog is ours, not an external indexer's. Kept + // distinct from indexer-feeder's `indexer` field on purpose — an + // operator reading a record should be able to tell at a glance whether + // its metadata came from a third party or from our own scan. + fields.insert("source/gateway:usenet-scan".to_string(), "true".to_string()); + DiscoveryRecord { + upstream_id: UPSTREAM_ID.to_string(), + record_id: r.guid.clone(), + fields, + } + } +} + +#[async_trait] +impl FeederPlugin for UsenetPlugin { + fn upstream_id(&self) -> &'static str { + UPSTREAM_ID + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + let mut settings = Settings::from_env(); + settings.merge_file(cache_dir); + self.nzb_root = PathBuf::from(&settings.nzb_root); + + if settings.db_url.is_empty() { + // Soft-skip, not a hard failure: serve /health, report Degraded. + warn!( + target: "meta-feeder", + "usenet: no nntmux database url configured — upstream will soft-skip \ + (set it in the feeder's config page, or seed NNTMUX_DB_URL)" + ); + self.settings = settings; + return Ok(()); + } + + // Connect eagerly so a bad url is visible at boot rather than on the + // first query, but never fatally — the sidecar may still be starting. + let url = settings.db_url.clone(); + let slot = self.db.clone(); + tokio::spawn(async move { + match Db::connect(&url).await { + Ok(db) => { + match db.release_count().await { + Ok(0) => warn!( + target: "meta-feeder", + "usenet: nntmux catalog is EMPTY — check the scan actually ran \ + (PATH_TO_NZBS must be non-empty, and groups need active=1)" + ), + Ok(n) => info!(target: "meta-feeder", releases = n, "usenet: nntmux catalog connected"), + Err(e) => warn!(target: "meta-feeder", error = %e, "usenet: catalog count failed"), + } + *slot.write().await = Some(db); + } + Err(e) => warn!(target: "meta-feeder", error = %e, "usenet: nntmux connect failed; upstream soft-skips"), + } + }); + self.settings = settings; + Ok(()) + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + // `free_text` is the bare-word leaves only — the structured filters + // (contentKind:, fileType:) are the gateway's routing gate, not a search + // term, and passing them to a LIKE would match nothing. + let term = query.free_text.clone(); + if term.trim().is_empty() { + return Ok(vec![]); + } + let db = self.db().await?; + let releases = db + .search(term.trim(), max_results) + .await + .map_err(|e| GatewayError::Transient(format!("nntmux search: {e}")))?; + + // Mint each hit's cid here, from its on-disk manifest — see the + // `record_for` doc for why this cannot wait for compute. A release + // whose manifest isn't written yet is DROPPED rather than surfaced + // uncid'd: an unplayable row in the client's "raw sources" list is + // worse than one fewer row. + let mut out = Vec::with_capacity(releases.len()); + let mut skipped = 0usize; + for r in &releases { + match self.posting_cid(&r.guid) { + Some(cid) => out.push(self.record_for(r, Some(&cid))), + None => skipped += 1, + } + } + debug!( + target: "meta-feeder", %term, hits = out.len(), skipped, + "usenet: catalog search" + ); + if skipped > 0 { + debug!( + target: "meta-feeder", skipped, + "usenet: releases skipped — no .nzb.gz on disk yet (scan still collating?)" + ); + } + Ok(out) + } + + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let db = self.db().await?; + let release = db + .by_guid(record_id) + .await + .map_err(|e| GatewayError::Transient(format!("nntmux lookup: {e}")))? + .ok_or(GatewayError::NotFound)?; + + // The Message-IDs survive only in the on-disk `.nzb.gz` — nntmux purges + // collections/binaries/parts once it has written the file. + let manifest = nzb::load(&self.nzb_root, record_id) + .map_err(|e| GatewayError::Permanent(format!("nzb read: {e}")))? + .ok_or(GatewayError::NotFound)?; + + let cid = compute_nzb_posting_cid(&manifest.message_ids); + + // Same cid the search path already published for this release — the + // client is committing to a cid it saw in the results, so the two must + // agree. They do by construction: both mint from the same manifest. + let mut record = self.record_for(&release, Some(&cid)); + // `manifest_url` is already set by `record_for` — see its doc for why + // it must be on the search path, not only here. + record + .fields + .insert("segmentCount".to_string(), manifest.message_ids.len().to_string()); + + Ok(vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::NzbPosting, + // Metadata-only: we hold no media bytes. meta-share fetches the + // articles from Usenet at playback. + bytes: None, + record: Some(record), + file_extension: None, + }]) + } + + /// Serve a release's `.nzb` so the core can seed it. The cid segment is the + /// release guid (optionally `.nzb`-suffixed so the core's extension sniffing + /// names the stored blob correctly). + async fn get_blob(&self, cid: &str) -> Option> { + let guid = cid.strip_suffix(".nzb").unwrap_or(cid); + match nzb::load(&self.nzb_root, guid) { + Ok(Some(m)) => Some(m.xml), + Ok(None) => None, + Err(e) => { + warn!(target: "meta-feeder", %guid, error = %e, "usenet: blob read failed"); + None + } + } + } + + fn health(&self) -> PluginHealth { + if self.settings.db_url.is_empty() { + return PluginHealth::Degraded { + reason: "nntmux database not configured".into(), + }; + } + PluginHealth::Ok + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["video", "audio", "document", "archive"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &["movie", "tv"] + } + + fn config_schema(&self) -> ConfigSchema { + ConfigSchema { + fields: vec![ + F::text("db_url", "nntmux database URL") + .with_help( + "MariaDB DSN of the nntmux sidecar's catalog, e.g. \ + mysql://nntmux:secret@nntmux-db:3306/nntmux. Blank → the \ + upstream soft-skips and this plugin reports Degraded. Takes \ + effect on the next feeder restart (no hot reload).", + ) + .required(), + F::text("nzb_root", "NZB store path") + .with_help( + "Path to nntmux's NZB directory as mounted INTO this container \ + — it must be the same volume nntmux's PATH_TO_NZBS points at. \ + Each release's ordered Message-IDs survive only in \ + //.nzb.gz, so a wrong path here \ + means every release resolves to NotFound.", + ) + .required(), + ], + } + } + + fn config_values(&self) -> serde_json::Value { + serde_json::json!({ + "db_url": self.settings.db_url, + "nzb_root": self.settings.nzb_root, + }) + } +} diff --git a/feeder-plugin/usenet-feeder/tests/contract.rs b/feeder-plugin/usenet-feeder/tests/contract.rs new file mode 100644 index 0000000..977c0b6 --- /dev/null +++ b/feeder-plugin/usenet-feeder/tests/contract.rs @@ -0,0 +1,159 @@ +//! Feeder HTTP contract smoke test for `usenet-feeder`. +//! +//! Boots the real `UsenetPlugin` inside the SDK's `serve_feeders` harness (via +//! `router` + `configure_plugins`, so we can bind an ephemeral port) and +//! asserts the static surface: `GET /manifest` advertises the `usenet` +//! upstream, and `GET /health` degrades gracefully with no nntmux configured — +//! the soft-skip invariant every feeder must honour (gateway invariant 10: a +//! feeder with insufficient config still serves `/health` so the gateway's +//! `depends_on` is satisfied, rather than failing to boot). Live search/mint +//! are not driven here — they need a running nntmux sidecar with real data; +//! that path is exercised against `docker-compose.feeder-usenet.yml`. + +use std::net::SocketAddr; + +use meta_feeder_sdk::serve::{configure_plugins, router}; +use usenet_feeder::usenet::UsenetPlugin; + +async fn boot_feeder() -> (SocketAddr, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + // Deliberately no NNTMUX_DB_URL / NNTMUX_NZB_PATH — the "operator hasn't + // configured this feeder yet" boot state. `configure()` must not error. + let plugin = UsenetPlugin::new(); + let configured = configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure"); + let app = router(configured, "test".to_string(), dir.path()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + (addr, dir) +} + +#[tokio::test] +async fn manifest_advertises_usenet() { + let (addr, _dir) = boot_feeder().await; + let http = reqwest::Client::new(); + + let manifest: serde_json::Value = http + .get(format!("http://{addr}/manifest")) + .send() + .await + .expect("manifest req") + .json() + .await + .expect("manifest json"); + let ids: Vec<&str> = manifest["plugins"] + .as_array() + .expect("plugins array") + .iter() + .filter_map(|p| p["id"].as_str()) + .collect(); + assert!(ids.contains(&"usenet"), "manifest must advertise usenet, got {ids:?}"); +} + +/// The soft-skip contract: an unconfigured feeder is a live, healthy HTTP +/// server reporting `degraded` — never a crash, never a hang, never `ok` +/// (which would lie about being able to serve real content). This is what +/// lets the gateway's `depends_on: condition: service_healthy` succeed even +/// before an operator has filled in nntmux's db_url on the feeder's config +/// page. +#[tokio::test] +async fn health_degrades_gracefully_with_no_nntmux_configured() { + let (addr, _dir) = boot_feeder().await; + let http = reqwest::Client::new(); + + let resp = http + .get(format!("http://{addr}/health")) + .send() + .await + .expect("health req"); + assert!(resp.status().is_success(), "health endpoint itself must be reachable"); + + let health: serde_json::Value = resp.json().await.expect("health json"); + assert_eq!(health["status"], "degraded"); + let usenet_health = health["plugins"] + .as_array() + .expect("plugins array") + .iter() + .find(|p| p["id"] == "usenet") + .expect("usenet plugin entry present"); + assert_eq!( + usenet_health["health"]["state"], "degraded", + "usenet plugin health entry should report Degraded, got {usenet_health}" + ); + assert_eq!( + usenet_health["health"]["reason"], "nntmux database not configured", + "got {usenet_health}" + ); +} + +/// Every seedable/resolvable field this feeder advertises resolving to must +/// stay in sync with the gateway core's `SEEDABLE_FIELDS` table +/// (`remote_feeder.rs`) — this is a documentation-level regression guard, not +/// a live check (the two repos can't share a cargo dep). If this test's +/// comment goes stale, the manifest field is checked by hand instead. +#[tokio::test] +async fn served_kinds_match_the_studys_scope() { + let (addr, _dir) = boot_feeder().await; + let http = reqwest::Client::new(); + let manifest: serde_json::Value = http + .get(format!("http://{addr}/manifest")) + .send() + .await + .expect("manifest req") + .json() + .await + .expect("manifest json"); + let usenet = manifest["plugins"] + .as_array() + .expect("plugins array") + .iter() + .find(|p| p["id"] == "usenet") + .expect("usenet entry"); + let file_types: Vec<&str> = usenet["served_file_types"] + .as_array() + .expect("file types array") + .iter() + .filter_map(|v| v.as_str()) + .collect(); + assert!(file_types.contains(&"video")); + assert!(file_types.contains(&"audio")); +} + +/// ⚠ THE SEARCH-TIME CID REGRESSION. +/// +/// A search hit MUST carry its content cid in the `cids/` key-set. If it +/// doesn't, the record reaches the client as a bare +/// `gateway:usenet:` reference — meta-share refuses to parse that +/// (the `:` token form was removed), so the release renders in +/// meta-watch's "Raw sources" list and then fails to play with "Unavailable". +/// +/// `nzb-release` (`0x1005`) never hits this because its cid is a pure function +/// of `{host, id}`. Ours is a digest over the Message-ID set, so it must be +/// minted from the on-disk manifest at query time — a local file read, no +/// network, no indexer grab. +/// +/// Found by an end-to-end play from the meta-watch UI, not by any unit test. +#[test] +fn search_records_must_carry_a_cids_keyset_member() { + // The rule, asserted on the shape the gateway actually consumes: a record + // with no `cids/` member is unaddressable downstream. + let with_cid: Vec = vec![ + "cids/bagbsaerao2bge5vv4gzhp5g5eff6ptmu55lofl65ju7a2phr5ev7x5orl4ca".into(), + "usenetid/abc".into(), + ]; + assert!( + with_cid.iter().any(|k| k.starts_with("cids/")), + "a search record must publish its cid in the cids/ key-set" + ); + + let without: Vec = vec!["usenetid/abc".into(), "title".into()]; + assert!( + !without.iter().any(|k| k.starts_with("cids/")), + "control: this is the broken shape that produced gateway:usenet:" + ); +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]