From 3509346db5ee4680b09ee4fa99118e8fd1e58af8 Mon Sep 17 00:00:00 2001 From: Worph Date: Sun, 14 Jun 2026 21:34:28 +0200 Subject: [PATCH] initial commit --- .github/workflows/publish.yml | 59 + Cargo.lock | 5688 +++++++++++++++++++ Cargo.toml | 9 + crates/meta-feeder-sdk/Cargo.toml | 49 + crates/meta-feeder-sdk/src/cache.rs | 337 ++ crates/meta-feeder-sdk/src/common.rs | 288 + crates/meta-feeder-sdk/src/config.rs | 312 + crates/meta-feeder-sdk/src/config_page.html | 245 + crates/meta-feeder-sdk/src/enrich.rs | 590 ++ crates/meta-feeder-sdk/src/filename_meta.rs | 1305 +++++ crates/meta-feeder-sdk/src/hash.rs | 521 ++ crates/meta-feeder-sdk/src/lang.rs | 81 + crates/meta-feeder-sdk/src/lib.rs | 42 + crates/meta-feeder-sdk/src/meta_core.rs | 293 + crates/meta-feeder-sdk/src/plugin.rs | 203 + crates/meta-feeder-sdk/src/query.rs | 147 + crates/meta-feeder-sdk/src/query_eval.rs | 456 ++ crates/meta-feeder-sdk/src/serve.rs | 634 +++ crates/meta-feeder-sdk/src/types.rs | 124 + feeder-plugin/book-feeder/Cargo.toml | 40 + feeder-plugin/book-feeder/Dockerfile | 55 + feeder-plugin/book-feeder/src/gutenberg.rs | 961 ++++ feeder-plugin/book-feeder/src/lib.rs | 5 + feeder-plugin/book-feeder/src/main.rs | 31 + feeder-plugin/book-feeder/tests/contract.rs | 172 + rust-toolchain.toml | 3 + 26 files changed, 12650 insertions(+) create mode 100644 .github/workflows/publish.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 crates/meta-feeder-sdk/Cargo.toml create mode 100644 crates/meta-feeder-sdk/src/cache.rs create mode 100644 crates/meta-feeder-sdk/src/common.rs create mode 100644 crates/meta-feeder-sdk/src/config.rs create mode 100644 crates/meta-feeder-sdk/src/config_page.html create mode 100644 crates/meta-feeder-sdk/src/enrich.rs create mode 100644 crates/meta-feeder-sdk/src/filename_meta.rs create mode 100644 crates/meta-feeder-sdk/src/hash.rs create mode 100644 crates/meta-feeder-sdk/src/lang.rs create mode 100644 crates/meta-feeder-sdk/src/lib.rs create mode 100644 crates/meta-feeder-sdk/src/meta_core.rs create mode 100644 crates/meta-feeder-sdk/src/plugin.rs create mode 100644 crates/meta-feeder-sdk/src/query.rs create mode 100644 crates/meta-feeder-sdk/src/query_eval.rs create mode 100644 crates/meta-feeder-sdk/src/serve.rs create mode 100644 crates/meta-feeder-sdk/src/types.rs create mode 100644 feeder-plugin/book-feeder/Cargo.toml create mode 100644 feeder-plugin/book-feeder/Dockerfile create mode 100644 feeder-plugin/book-feeder/src/gutenberg.rs create mode 100644 feeder-plugin/book-feeder/src/lib.rs create mode 100644 feeder-plugin/book-feeder/src/main.rs create mode 100644 feeder-plugin/book-feeder/tests/contract.rs create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..43ac688 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/book-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/book-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..d4b53ec --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5688 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array 0.14.7", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "assert_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e2651f366b7ee3f97729fded1441539b49d5f39eeb05b842689e11e84501b2" +dependencies = [ + "const_panic", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "asynchronous-codec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64", + "http", + "log", + "url", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.17", + "instant", + "rand 0.8.6", +] + +[[package]] +name = "base-x" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" + +[[package]] +name = "base256emoji" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e9430d9a245a77c92176e649af6e275f20839a48389859d1661e9a128d077c" +dependencies = [ + "const-str", + "match-lookup", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "beetswap" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a920844a4696d53e72190a8348f0048a6799647b3448d38bef90b85e0310df0" +dependencies = [ + "asynchronous-codec", + "blockstore", + "bytes", + "cid", + "fnv", + "futures-core", + "futures-timer", + "futures-util", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "multihash-codetable", + "quick-protobuf", + "smallvec", + "thiserror 2.0.18", + "tracing", + "unsigned-varint 0.8.0", + "web-time", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake2b_simd" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake2s_simd" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee29928bad1e3f94c9d1528da29e07a1d3d04817ae8332de1e8b846c8439f4b3" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array 0.14.7", +] + +[[package]] +name = "blockstore" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509096e88e431095763b3f5ee1e1cdb09212e4d5b2eccc91ddea965deefedb7d" +dependencies = [ + "cid", + "dashmap", + "multihash", + "redb", + "thiserror 2.0.18", + "tokio", +] + +[[package]] +name = "book-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "meta-feeder-sdk", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cid" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a304f95f84d169a6f31c4d0a30d784643aaa0bbc9c1e449a2c23e963ec4971" +dependencies = [ + "multibase", + "multihash", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", + "zeroize", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "common-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "meta-feeder-sdk", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-str" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" + +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core2" +version = "0.4.0" +source = "git+https://github.com/bbqsrc/core2?rev=545e84b#545e84bcb0f235b12e21351e0c69767958efe2a7" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array 0.14.7", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "serde", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "data-encoding-macro" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +dependencies = [ + "data-encoding", + "data-encoding-macro-internal", +] + +[[package]] +name = "data-encoding-macro-internal" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +dependencies = [ + "data-encoding", + "syn", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-bounded" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f328e7fb845fc832912fb6a34f40cf6d1888c92f974d1893a54e97b5ff542e" +dependencies = [ + "futures-timer", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f2f12607f92c69b12ed746fabf9ca4f5c482cba46679c1a75b874ed7c26adb" +dependencies = [ + "futures-io", + "rustls", + "rustls-pki-types", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.4", + "smallvec", + "spinning_top", + "web-time", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hex_fmt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" + +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.4", + "ring", + "socket2 0.5.10", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.4", + "resolv-conf", + "smallvec", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.3", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "if-watch" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c02a5161c313f0cbdbadc511611893584a10a7b6153cb554bdf83ddce99ec2" +dependencies = [ + "async-io", + "core-foundation", + "fnv", + "futures", + "if-addrs", + "ipnet", + "log", + "netlink-packet-core", + "netlink-packet-route", + "netlink-proto", + "netlink-sys", + "rtnetlink", + "system-configuration", + "tokio", + "windows", +] + +[[package]] +name = "igd-next" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516893339c97f6011282d5825ac94fc1c7aad5cad26bdc2d0cee068c0bf97f97" +dependencies = [ + "async-trait", + "attohttpc", + "bytes", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "rand 0.9.4", + "tokio", + "url", + "xmltree", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array 0.14.7", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "intervaltree" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "270bc34e57047cab801a8c871c124d9dc7132f6473c6401f645524f4e6edd111" +dependencies = [ + "smallvec", +] + +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2 0.6.3", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leaky-bucket" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a396bb213c2d09ed6c5495fd082c991b6ab39c9daf4fff59e6727f85c73e4c5" +dependencies = [ + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libp2p" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce71348bf5838e46449ae240631117b487073d5f347c06d434caddcb91dceb5a" +dependencies = [ + "bytes", + "either", + "futures", + "futures-timer", + "getrandom 0.2.17", + "libp2p-allow-block-list", + "libp2p-connection-limits", + "libp2p-core", + "libp2p-dns", + "libp2p-gossipsub", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-mdns", + "libp2p-metrics", + "libp2p-noise", + "libp2p-ping", + "libp2p-quic", + "libp2p-request-response", + "libp2p-swarm", + "libp2p-tcp", + "libp2p-upnp", + "libp2p-yamux", + "multiaddr", + "pin-project", + "rw-stream-sink", + "thiserror 2.0.18", +] + +[[package]] +name = "libp2p-allow-block-list" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16ccf824ee859ca83df301e1c0205270206223fd4b1f2e512a693e1912a8f4a" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", +] + +[[package]] +name = "libp2p-connection-limits" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a18b8b607cf3bfa2f8c57db9c7d8569a315d5cc0a282e6bfd5ebfc0a9840b2a0" +dependencies = [ + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", +] + +[[package]] +name = "libp2p-core" +version = "0.43.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249128cd37a2199aff30a7675dffa51caf073b51aa612d2f544b19932b9aebca" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "libp2p-identity", + "multiaddr", + "multihash", + "multistream-select", + "parking_lot", + "pin-project", + "quick-protobuf", + "rand 0.8.6", + "rw-stream-sink", + "thiserror 2.0.18", + "tracing", + "unsigned-varint 0.8.0", + "web-time", +] + +[[package]] +name = "libp2p-dns" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b770c1c8476736ca98c578cba4b505104ff8e842c2876b528925f9766379f9a" +dependencies = [ + "async-trait", + "futures", + "hickory-resolver", + "libp2p-core", + "libp2p-identity", + "parking_lot", + "smallvec", + "tracing", +] + +[[package]] +name = "libp2p-gossipsub" +version = "0.49.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a538e571cd38f504f761c61b8f79127489ea7a7d6f05c41ca15d31ffb5726326" +dependencies = [ + "async-channel", + "asynchronous-codec", + "base64", + "byteorder", + "bytes", + "either", + "fnv", + "futures", + "futures-timer", + "getrandom 0.2.17", + "hashlink 0.9.1", + "hex_fmt", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "quick-protobuf", + "quick-protobuf-codec", + "rand 0.8.6", + "regex", + "sha2", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-identify" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab792a8b68fdef443a62155b01970c81c3aadab5e659621b063ef252a8e65e8" +dependencies = [ + "asynchronous-codec", + "either", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "quick-protobuf", + "quick-protobuf-codec", + "smallvec", + "thiserror 2.0.18", + "tracing", +] + +[[package]] +name = "libp2p-identity" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c7892c221730ba55f7196e98b0b8ba5e04b4155651736036628e9f73ed6fc3" +dependencies = [ + "bs58", + "ed25519-dalek", + "hkdf", + "multihash", + "quick-protobuf", + "rand 0.8.6", + "sha2", + "thiserror 2.0.18", + "tracing", + "zeroize", +] + +[[package]] +name = "libp2p-kad" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d3fd632a5872ec804d37e7413ceea20588f69d027a0fa3c46f82574f4dee60" +dependencies = [ + "asynchronous-codec", + "bytes", + "either", + "fnv", + "futures", + "futures-bounded", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "quick-protobuf", + "quick-protobuf-codec", + "rand 0.8.6", + "sha2", + "smallvec", + "thiserror 2.0.18", + "tracing", + "uint", + "web-time", +] + +[[package]] +name = "libp2p-mdns" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66872d0f1ffcded2788683f76931be1c52e27f343edb93bc6d0bcd8887be443" +dependencies = [ + "futures", + "hickory-proto", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.6", + "smallvec", + "socket2 0.5.10", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-metrics" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805a555148522cb3414493a5153451910cb1a146c53ffbf4385708349baf62b7" +dependencies = [ + "futures", + "libp2p-core", + "libp2p-gossipsub", + "libp2p-identify", + "libp2p-identity", + "libp2p-kad", + "libp2p-ping", + "libp2p-swarm", + "pin-project", + "prometheus-client", + "web-time", +] + +[[package]] +name = "libp2p-noise" +version = "0.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc73eacbe6462a0eb92a6527cac6e63f02026e5407f8831bde8293f19217bfbf" +dependencies = [ + "asynchronous-codec", + "bytes", + "futures", + "libp2p-core", + "libp2p-identity", + "multiaddr", + "multihash", + "quick-protobuf", + "rand 0.8.6", + "snow", + "static_assertions", + "thiserror 2.0.18", + "tracing", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "libp2p-ping" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74bb7fcdfd9fead4144a3859da0b49576f171a8c8c7c0bfc7c541921d25e60d3" +dependencies = [ + "futures", + "futures-timer", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.6", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-quic" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc448b2de9f4745784e3751fe8bc6c473d01b8317edd5ababcb0dec803d843f" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libp2p-core", + "libp2p-identity", + "libp2p-tls", + "quinn", + "rand 0.8.6", + "ring", + "rustls", + "socket2 0.5.10", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-request-response" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f1cca83488b90102abac7b67d5c36fc65bc02ed47620228af7ed002e6a1478" +dependencies = [ + "async-trait", + "futures", + "futures-bounded", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.6", + "smallvec", + "tracing", +] + +[[package]] +name = "libp2p-stream" +version = "0.4.0-alpha" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6bd8025c80205ec2810cfb28b02f362ab48a01bee32c50ab5f12761e033464" +dependencies = [ + "futures", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm", + "rand 0.8.6", + "tracing", +] + +[[package]] +name = "libp2p-swarm" +version = "0.47.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce88c6c4bf746c8482480345ea3edfd08301f49e026889d1cbccfa1808a9ed9e" +dependencies = [ + "either", + "fnv", + "futures", + "futures-timer", + "hashlink 0.10.0", + "libp2p-core", + "libp2p-identity", + "libp2p-swarm-derive", + "multistream-select", + "rand 0.8.6", + "smallvec", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "libp2p-swarm-derive" +version = "0.35.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" +dependencies = [ + "heck", + "quote", + "syn", +] + +[[package]] +name = "libp2p-tcp" +version = "0.44.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6585b9309699f58704ec9ab0bb102eca7a3777170fa91a8678d73ca9cafa93" +dependencies = [ + "futures", + "futures-timer", + "if-watch", + "libc", + "libp2p-core", + "socket2 0.6.3", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-tls" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96ff65a82e35375cbc31ebb99cacbbf28cb6c4fefe26bf13756ddcf708d40080" +dependencies = [ + "futures", + "futures-rustls", + "libp2p-core", + "libp2p-identity", + "rcgen", + "ring", + "rustls", + "rustls-webpki", + "thiserror 2.0.18", + "x509-parser", + "yasna", +] + +[[package]] +name = "libp2p-upnp" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4757e65fe69399c1a243bbb90ec1ae5a2114b907467bf09f3575e899815bb8d3" +dependencies = [ + "futures", + "futures-timer", + "igd-next", + "libp2p-core", + "libp2p-swarm", + "tokio", + "tracing", +] + +[[package]] +name = "libp2p-yamux" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f15df094914eb4af272acf9adaa9e287baa269943f32ea348ba29cfb9bfc60d8" +dependencies = [ + "either", + "futures", + "libp2p-core", + "thiserror 2.0.18", + "tracing", + "yamux 0.12.1", + "yamux 0.13.10", +] + +[[package]] +name = "libredox" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" +dependencies = [ + "libc", +] + +[[package]] +name = "librqbit" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dadca8f521242010a4c846ef5f224c217009c92e272709cdc08ba9cdabe62983" +dependencies = [ + "anyhow", + "arc-swap", + "async-compression", + "async-stream", + "async-trait", + "backoff", + "base64", + "bincode 2.0.1", + "bitvec", + "byteorder", + "bytes", + "dashmap", + "futures", + "governor", + "hex", + "http", + "intervaltree", + "itertools", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "librqbit-dht", + "librqbit-peer-protocol", + "librqbit-sha1-wrapper", + "librqbit-tracker-comms", + "librqbit-upnp", + "memmap2", + "mime_guess", + "parking_lot", + "rand 0.9.4", + "regex", + "reqwest", + "rlimit", + "serde", + "serde_json", + "serde_urlencoded", + "serde_with", + "size_format", + "tokio", + "tokio-socks", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "urlencoding", + "uuid", + "walkdir", +] + +[[package]] +name = "librqbit-bencode" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606dff526ba81e3eca33e2bb28b53afa2bc0b2c41d252333fa44e6c11abb37da" +dependencies = [ + "anyhow", + "bytes", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-sha1-wrapper", + "serde", +] + +[[package]] +name = "librqbit-buffers" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c78b907d6171a7191c162b2b60db46d254ebde6a95282b77372af556c1463" +dependencies = [ + "bytes", + "librqbit-clone-to-owned", + "serde", +] + +[[package]] +name = "librqbit-clone-to-owned" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1e66d773ba9c475ff89286dc1d6f9d167cbb898603797467dd0ea6844c445" +dependencies = [ + "bytes", +] + +[[package]] +name = "librqbit-core" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a02cc6fce6743ad38661ccd6fafc6cf1ae5e0106a9922836b0524dbe752378" +dependencies = [ + "anyhow", + "assert_cfg", + "bytes", + "data-encoding", + "directories", + "hex", + "itertools", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "parking_lot", + "rand 0.9.4", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "librqbit-dht" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7cc129194337771a86b0399956c4d9bf1cd97c5f24d14a50be38e170f76a54b" +dependencies = [ + "anyhow", + "backoff", + "byteorder", + "bytes", + "chrono", + "dashmap", + "futures", + "hex", + "indexmap 2.14.0", + "leaky-bucket", + "librqbit-bencode", + "librqbit-clone-to-owned", + "librqbit-core", + "parking_lot", + "rand 0.9.4", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + +[[package]] +name = "librqbit-peer-protocol" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a73129497b500505f33d1dc0426319b6a6a208f13fdfaae56224ab8c2346a773" +dependencies = [ + "anyhow", + "bincode 1.3.3", + "bitvec", + "byteorder", + "bytes", + "itertools", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "serde", +] + +[[package]] +name = "librqbit-sha1-wrapper" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79373a02db73159e4de7ca5d27b6eeae2d540df66c6801db2b01c5513d087524" +dependencies = [ + "assert_cfg", + "aws-lc-rs", +] + +[[package]] +name = "librqbit-tracker-comms" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08204944c5be677a5de8e1230e0249fce5c14abef23048e26452c6fb03f1b260" +dependencies = [ + "anyhow", + "async-stream", + "byteorder", + "futures", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-core", + "parking_lot", + "rand 0.9.4", + "reqwest", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", + "urlencoding", +] + +[[package]] +name = "librqbit-upnp" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545aad6124c97201055983137e12a19f34acad565120c3cd30596cbd72e8fa86" +dependencies = [ + "anyhow", + "bstr", + "futures", + "httparse", + "network-interface", + "quick-xml 0.37.5", + "reqwest", + "serde", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "match-lookup" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "meta-gateway" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "beetswap", + "bincode 2.0.1", + "blockstore", + "bytes", + "chrono", + "cid", + "futures", + "hostname", + "hyper", + "libp2p", + "libp2p-stream", + "meta-feeder-sdk", + "redb", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tower-http", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multiaddr" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe6351f60b488e04c1d21bc69e56b89cb3f5e8f5d22557d6e8031bdfd79b6961" +dependencies = [ + "arrayref", + "byteorder", + "data-encoding", + "libp2p-identity", + "multibase", + "multihash", + "percent-encoding", + "serde", + "static_assertions", + "unsigned-varint 0.8.0", + "url", +] + +[[package]] +name = "multibase" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8694bb4835f452b0e3bb06dbebb1d6fc5385b6ca1caf2e55fd165c042390ec77" +dependencies = [ + "base-x", + "base256emoji", + "data-encoding", + "data-encoding-macro", +] + +[[package]] +name = "multihash" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c63b00ad74d57e8c9aa870b5fccebf2fd64a308a5aee9f1bb88e4aea19447" +dependencies = [ + "unsigned-varint 0.8.0", +] + +[[package]] +name = "multihash-codetable" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67996849749d25f1da9f238e8ace2ece8f9d6bdf3f9750aaf2ae7de3a5cad8ea" +dependencies = [ + "blake2b_simd", + "blake2s_simd", + "blake3", + "core2", + "digest", + "multihash-derive", + "ripemd", + "sha1", + "sha2", + "sha3", + "strobe-rs", +] + +[[package]] +name = "multihash-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4723acdce756db211a53f01b3419dbdf63cb48cef5df86260f55309364735fbf" +dependencies = [ + "multihash", + "multihash-derive-impl", +] + +[[package]] +name = "multihash-derive-impl" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f932556f78452e5604cef711349d337ec081a9aa3c96e67b3127c8f5df05d550" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "multistream-select" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" +dependencies = [ + "bytes", + "futures", + "log", + "pin-project", + "smallvec", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "netlink-packet-core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3463cbb78394cb0141e2c926b93fc2197e473394b761986eca3b9da2c63ae0f4" +dependencies = [ + "paste", +] + +[[package]] +name = "netlink-packet-route" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce3636fa715e988114552619582b530481fd5ef176a1e5c1bf024077c2c9445" +dependencies = [ + "bitflags 2.11.1", + "libc", + "log", + "netlink-packet-core", +] + +[[package]] +name = "netlink-proto" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65d130ee111430e47eed7896ea43ca693c387f097dd97376bffafbf25812128" +dependencies = [ + "bytes", + "futures", + "log", + "netlink-packet-core", + "netlink-sys", + "thiserror 2.0.18", +] + +[[package]] +name = "netlink-sys" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6c30ed10fa69cc491d491b85cc971f6bdeb8e7367b7cde2ee6cc878d583fae" +dependencies = [ + "bytes", + "futures-util", + "libc", + "log", + "tokio", +] + +[[package]] +name = "network-interface" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddcb8865ad3d9950f22f42ffa0ef0aecbfbf191867b3122413602b0a360b2a6" +dependencies = [ + "cc", + "libc", + "thiserror 2.0.18", + "winapi", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "paper-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "futures", + "meta-feeder-sdk", + "quick-xml 0.36.2", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus-client" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf41c1a7c32ed72abe5082fb19505b969095c12da9f5732a4bc9878757fd087c" +dependencies = [ + "dtoa", + "itoa", + "parking_lot", + "prometheus-client-derive-encode", +] + +[[package]] +name = "prometheus-client-derive-encode" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quick-protobuf" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f" +dependencies = [ + "byteorder", +] + +[[package]] +name = "quick-protobuf-codec" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474" +dependencies = [ + "asynchronous-codec", + "bytes", + "quick-protobuf", + "thiserror 1.0.69", + "unsigned-varint 0.8.0", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "futures-io", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.3", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.3", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "ripemd" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" +dependencies = [ + "digest", +] + +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + +[[package]] +name = "rtnetlink" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b960d5d873a75b5be9761b1e73b146f52dddcd27bac75263f40fba686d4d7b5" +dependencies = [ + "futures-channel", + "futures-util", + "log", + "netlink-packet-core", + "netlink-packet-route", + "netlink-proto", + "netlink-sys", + "nix", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted 0.9.0", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rw-stream-sink" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" +dependencies = [ + "futures", + "pin-project", + "static_assertions", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +dependencies = [ + "base64", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "size_format" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed5f6ab2122c6dec69dca18c72fa4590a27e581ad20d44960fe74c032a0b23b" +dependencies = [ + "generic-array 0.12.4", + "num", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snow" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85" +dependencies = [ + "aes-gcm", + "blake2", + "chacha20poly1305", + "curve25519-dalek", + "rand_core 0.6.4", + "ring", + "rustc_version", + "sha2", + "subtle", +] + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strobe-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98fe17535ea31344936cc58d29fec9b500b0452ddc4cc24c429c8a921a0e84e5" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "keccak", + "subtle", + "zeroize", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.1", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.3", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-socks" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +dependencies = [ + "either", + "futures-util", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap 2.14.0", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "torrent-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "chrono", + "futures", + "librqbit", + "meta-feeder-sdk", + "quick-xml 0.36.2", + "regex", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" + +[[package]] +name = "uint" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.14.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.1", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +dependencies = [ + "memchr", +] + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.14.0", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.1", + "indexmap 2.14.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.14.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core 0.6.4", + "serde", + "zeroize", +] + +[[package]] +name = "x509-parser" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "xml-rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" + +[[package]] +name = "xmltree" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" +dependencies = [ + "xml-rs", +] + +[[package]] +name = "yamux" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed0164ae619f2dc144909a9f082187ebb5893693d8c0196e8085283ccd4b776" +dependencies = [ + "futures", + "log", + "nohash-hasher", + "parking_lot", + "pin-project", + "rand 0.8.6", + "static_assertions", +] + +[[package]] +name = "yamux" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1991f6690292030e31b0144d73f5e8368936c58e45e7068254f7138b23b00672" +dependencies = [ + "futures", + "log", + "nohash-hasher", + "parking_lot", + "pin-project", + "rand 0.9.4", + "static_assertions", + "web-time", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3a8de4b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["crates/*", "feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-book" diff --git a/crates/meta-feeder-sdk/Cargo.toml b/crates/meta-feeder-sdk/Cargo.toml new file mode 100644 index 0000000..bbe98ff --- /dev/null +++ b/crates/meta-feeder-sdk/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "meta-feeder-sdk" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Source-agnostic foundation shared by the gateway core and every feeder +# sidecar binary: the `FeederPlugin` trait, the content-addressing hash +# families, the per-plugin redb cache, the shared HTTP/cache helpers, the +# structured-query evaluator, and the axum `serve_feeders` harness that wraps +# any set of `FeederPlugin`s in the feeder HTTP contract. +# +# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes; +# the gateway core owns hashing-into-the-blockstore, the meta-core store-back, +# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT +# of here so feeders that don't need them don't pay for them. + +[dependencies] +# Async runtime + HTTP server (the serve_feeders harness) +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +hyper = { version = "1", features = ["full"] } +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" +base64 = "0.22" + +# Utilities +futures = "0.3" +async-trait = "0.1" +anyhow = "1" +thiserror = "1" +tracing = "0.1" +sha2 = "0.10" +bytes = "1" +# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). +regex = "1" + +# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see +# cache.rs. Pure-Rust single-process embedded store. +redb = "2" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/crates/meta-feeder-sdk/src/cache.rs b/crates/meta-feeder-sdk/src/cache.rs new file mode 100644 index 0000000..f22e99e --- /dev/null +++ b/crates/meta-feeder-sdk/src/cache.rs @@ -0,0 +1,337 @@ +//! Per-plugin midhash cache backed by redb. +//! +//! Each plugin gets its own redb file at +//! `/gateway//cache.redb`. Keys are upstream +//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the +//! `Midhash` strings the plugin computed by fetching + hashing the upstream +//! file. +//! +//! The schema is identical to meta-share v1's so old `.redb` files copy +//! across. redb is synchronous and sub-millisecond for small ops, so calls +//! run inline from async handlers without `spawn_blocking`. + +use std::path::Path; +use std::sync::Arc; + +use redb::{Database, ReadableTableMetadata, TableDefinition}; + +pub const CACHE_FILENAME: &str = "cache.redb"; + +const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash"); +const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs"); +const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid"); +const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec"); +const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid"); +/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload +/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the +/// hash families differ: `midhash` is the synthetic midhash256-from-infohash +/// fallback; `fullhash` records "we actually downloaded the bytes and this is +/// the IPFS CIDv1 over them." +const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash"); +/// Enumerated torrent file list (JSON `Vec`) keyed by the +/// torrent's `record_id` (infohash). +const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist"); +/// Cached TMDB *search* results keyed by a stable lookup key. The value is the +/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache +/// sentinel. +const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search"); +/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`. +const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_tvdetails"); +/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`. +const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids"); +/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`. +const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_moviedetails"); +/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized +/// free-text query. +const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_principal_topn"); +/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's +/// `record_id` (infohash). Value is the JSON encoding of a `Vec`. +const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles"); +/// OpenSubtitles search results keyed by `"\x01"`. Value is the +/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel. +const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles"); + +/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared +/// via `Arc`). +#[derive(Clone)] +pub struct MidhashCache { + db: Arc, +} + +/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb +/// table. +macro_rules! str_table_accessors { + ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => { + $(#[$gmeta])* + pub fn $get(&self, key: &str) -> Result, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table($table)?; + Ok(table.get(key)?.map(|v| v.value().to_string())) + } + + $(#[$pmeta])* + pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table($table)?; + table.insert(key, value)?; + } + tx.commit()?; + Ok(()) + } + }; +} + +impl MidhashCache { + pub fn open(cache_dir: &Path) -> Result { + let path = cache_dir.join(CACHE_FILENAME); + let db = Database::create(&path)?; + { + let tx = db.begin_write()?; + tx.open_table(MIDHASH_TABLE)?; + tx.open_table(BLOBS_TABLE)?; + tx.open_table(COVER_CID_TABLE)?; + tx.open_table(BIBREC_TABLE)?; + tx.open_table(PREVIEW_CID_TABLE)?; + tx.open_table(FULLHASH_TABLE)?; + tx.open_table(FILELIST_TABLE)?; + tx.open_table(TMDB_SEARCH_TABLE)?; + tx.open_table(TMDB_TVDETAILS_TABLE)?; + tx.open_table(TMDB_EXTIDS_TABLE)?; + tx.open_table(TMDB_MOVIEDETAILS_TABLE)?; + tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?; + tx.open_table(SUBTITLES_TABLE)?; + tx.open_table(OPENSUBTITLES_TABLE)?; + tx.commit()?; + } + Ok(Self { db: Arc::new(db) }) + } + + str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE); + + pub fn entry_count(&self) -> Result { + let tx = self.db.begin_read()?; + let table = tx.open_table(MIDHASH_TABLE)?; + Ok(table.len()?) + } + + pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BLOBS_TABLE)?; + Ok(table.get(cid)?.map(|v| v.value().to_vec())) + } + + pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BLOBS_TABLE)?; + table.insert(cid, bytes)?; + } + tx.commit()?; + Ok(()) + } + + str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE); + + str_table_accessors!( + /// Read a previously-stored full-file IPFS cid for `record_id`. + get_fullhash, + /// Record the IPFS cid produced by a successful full fetch. + put_fullhash, + FULLHASH_TABLE + ); + + str_table_accessors!( + /// Read the cached torrent file list (JSON) for `record_id`. + get_filelist, + /// Cache the enumerated torrent file list (JSON `Vec`). + put_filelist, + FILELIST_TABLE + ); + + str_table_accessors!( + /// Read the cached subtitle linkage (JSON `Vec`). + get_subtitles, + /// Cache the discovered subtitle linkage (JSON `Vec`). + put_subtitles, + SUBTITLES_TABLE + ); + + str_table_accessors!( + /// Read a cached OpenSubtitles lookup by `"\x01"`. + get_opensubtitles, + /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel). + put_opensubtitles, + OPENSUBTITLES_TABLE + ); + + str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE); + + str_table_accessors!( + /// Read a cached TMDB search result by lookup key. + get_tmdb_search, + /// Cache a TMDB search result under `key`. + put_tmdb_search, + TMDB_SEARCH_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB TV-details JSON by `tmdbid`. + get_tmdb_tvdetails, + /// Cache TMDB TV-details JSON under `tmdbid`. + put_tmdb_tvdetails, + TMDB_TVDETAILS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB external-ids JSON by `tmdbid`. + get_tmdb_extids, + /// Cache TMDB external-ids JSON under `tmdbid`. + put_tmdb_extids, + TMDB_EXTIDS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB movie-details JSON by `tmdbid`. + get_tmdb_moviedetails, + /// Cache TMDB movie-details JSON under `tmdbid`. + put_tmdb_moviedetails, + TMDB_MOVIEDETAILS_TABLE + ); + + str_table_accessors!( + /// Read the cached ranked anchor list by normalized query key. + get_tmdb_principal_topn, + /// Cache the ranked anchor list under the normalized query key. + put_tmdb_principal_topn, + TMDB_PRINCIPAL_TOPN_TABLE + ); + + pub fn get_bibrec( + &self, + record_id: &str, + ) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BIBREC_TABLE)?; + let Some(v) = table.get(record_id)? else { + return Ok(None); + }; + let s = v.value(); + Ok(serde_json::from_str(s).ok()) + } + + pub fn put_bibrec( + &self, + record_id: &str, + fields: &std::collections::BTreeMap, + ) -> Result<(), redb::Error> { + let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()); + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BIBREC_TABLE)?; + table.insert(record_id, json.as_str())?; + } + tx.commit()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_cache_dir() -> tempfile::TempDir { + tempfile::tempdir().expect("tempdir") + } + + #[test] + fn put_then_get_returns_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None); + cache.put_midhash("10.1038/x", "bafyXYZ").expect("put"); + assert_eq!( + cache.get_midhash("10.1038/x").unwrap().as_deref(), + Some("bafyXYZ") + ); + } + + #[test] + fn put_overwrites_existing_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + cache.put_midhash("k", "v1").unwrap(); + cache.put_midhash("k", "v2").unwrap(); + assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2")); + } + + #[test] + fn values_persist_across_reopens() { + let dir = tmp_cache_dir(); + { + let cache = MidhashCache::open(dir.path()).expect("open 1"); + cache.put_midhash("k", "persistent").unwrap(); + } + let cache = MidhashCache::open(dir.path()).expect("open 2"); + assert_eq!( + cache.get_midhash("k").unwrap().as_deref(), + Some("persistent") + ); + } + + #[test] + fn tmdb_search_cache_distinguishes_miss_from_absent() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None); + cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap(); + assert_eq!( + cache + .get_tmdb_search("m\u{1}naruto\u{1}") + .unwrap() + .as_deref(), + Some("null") + ); + cache + .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}") + .unwrap(); + drop(cache); + let cache = MidhashCache::open(dir.path()).expect("reopen"); + assert_eq!( + cache + .get_tmdb_search("t\u{1}spy x family\u{1}2019") + .unwrap() + .as_deref(), + Some("{\"tmdbid\":120089}") + ); + } + + #[test] + fn tmdb_tvdetails_cache_roundtrips() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None); + cache + .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}") + .unwrap(); + assert_eq!( + cache.get_tmdb_tvdetails("120089").unwrap().as_deref(), + Some("{\"number_of_seasons\":2}") + ); + } + + #[test] + fn entry_count_reflects_inserts() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.entry_count().unwrap(), 0); + cache.put_midhash("a", "x").unwrap(); + cache.put_midhash("b", "y").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + cache.put_midhash("a", "x2").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + } +} diff --git a/crates/meta-feeder-sdk/src/common.rs b/crates/meta-feeder-sdk/src/common.rs new file mode 100644 index 0000000..c7a09f5 --- /dev/null +++ b/crates/meta-feeder-sdk/src/common.rs @@ -0,0 +1,288 @@ +//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`, +//! `pubmed`, `wikicommons`, `giphy`, `scihub`). +//! +//! These plugins are structurally near-identical: build a reqwest client, map +//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose +//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding +//! lives here; the genuinely-different bodies stay in the plugin modules. + +use std::path::Path; +use std::time::Duration; + +use tracing::{debug, warn}; + +use crate::cache::MidhashCache; +use crate::plugin::{ConfigError, HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, GatewayError, Hash}; + +/// Build the shared reqwest client every source plugin uses. +pub fn build_http_client( + timeout_secs: u64, + user_agent: &str, + redirect: Option, +) -> reqwest::Client { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(timeout_secs)) + .user_agent(user_agent); + if let Some(policy) = redirect { + builder = builder.redirect(policy); + } + builder + .build() + .expect("reqwest client build is infallible with rustls-tls") +} + +/// Map a non-success HTTP status to a [`GatewayError`]. +/// +/// - `404` → [`GatewayError::NotFound`] +/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s) +/// - `5xx` → [`GatewayError::Transient`] +/// - anything else → [`GatewayError::Permanent`] +pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> { + let status = resp.status(); + if status.is_success() { + return Ok(()); + } + let code = status.as_u16(); + if code == 404 { + return Err(GatewayError::NotFound); + } + if code == 429 { + let retry_after = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(60); + return Err(GatewayError::RateLimited { + retry_after_s: retry_after, + }); + } + if status.is_server_error() { + return Err(GatewayError::Transient(format!( + "upstream {} returned {status}", + resp.url() + ))); + } + Err(GatewayError::Permanent(format!( + "upstream {} returned {status}", + resp.url() + ))) +} + +/// Open the per-plugin redb midhash cache, wrapping the error with the plugin +/// name for a clean startup diagnostic. +pub fn open_midhash_cache( + cache_dir: &Path, + plugin: &'static str, +) -> Result { + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin, + source: anyhow::anyhow!("open redb cache: {e}"), + }) +} + +/// Cache-hit fast path for single-outcome plugins. +/// +/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record — +/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a +/// cache miss; the caller proceeds to fetch + hash. +pub fn cached_outcome( + cache: &MidhashCache, + record_id: &str, + upstream_id: &'static str, +) -> Result>, GatewayError> { + let Some(cached) = cache + .get_midhash(record_id) + .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))? + else { + return Ok(None); + }; + debug!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + "hash cache hit" + ); + Ok(Some(vec![HashOutcome { + hash: Hash(cached), + hash_kind: HashKind::Sha2_256, + bytes: None, + record: None, + file_extension: None, + }])) +} + +/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is +/// non-fatal, so it is logged and swallowed. +pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) { + if let Err(e) = cache.put_midhash(record_id, cid) { + warn!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + error = %e, + "hash cache put failed (non-fatal)" + ); + } +} + +/// Resolve the per-plugin cache handle, or produce the standard "not +/// configured" internal error. +pub fn require_cache<'a>( + cache: Option<&'a MidhashCache>, + plugin: &str, +) -> Result<&'a MidhashCache, GatewayError> { + cache.ok_or_else(|| { + GatewayError::Internal(anyhow::anyhow!( + "{plugin} plugin not configured (configure() never called)" + )) + }) +} + +/// Build the single-element outcome vec a single-outcome plugin returns on a +/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID, +/// and an optional file extension. +pub fn single_outcome( + cid: String, + bytes: bytes::Bytes, + record: DiscoveryRecord, + file_extension: Option, +) -> Vec { + vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record: Some(record), + file_extension, + }] +} + +/// Percent-encode a query-string value using the unreserved set +/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space. +pub fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.as_bytes() { + match *b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*b as char); + } + b' ' => out.push('+'), + other => out.push_str(&format!("%{:02X}", other)), + } + } + out +} + +/// GET `url` and return the body bytes, mapping transport + status errors to +/// [`GatewayError`]. +pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.bytes() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Like [`fetch_bytes`] but decodes the body as UTF-8 text. +pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.text() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Decode a v1 BitTorrent infohash string into its raw 20 bytes. +/// +/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648, +/// case-insensitive). Returns `None` for anything else. +pub fn decode_infohash(s: &str) -> Option<[u8; 20]> { + let bytes = s.as_bytes(); + let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) { + let mut out = Vec::with_capacity(20); + for pair in bytes.chunks(2) { + out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?); + } + out + } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) { + decode_base32_20(bytes)? + } else { + return None; + }; + raw.try_into().ok() +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20 +/// bytes they encode. +fn decode_base32_20(chars: &[u8]) -> Option> { + let mut out = Vec::with_capacity(20); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &c in chars { + let val = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a', + b'2'..=b'7' => c - b'2' + 26, + _ => return None, + } as u64; + buffer = (buffer << 5) | val; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push((buffer >> bits) as u8); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_infohash_hex_roundtrips() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let raw = decode_infohash(hex).expect("hex decodes"); + assert_eq!(raw[0], 0x01); + assert_eq!(raw[19], 0x67); + } + + #[test] + fn decode_infohash_is_case_insensitive_hex() { + let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd"); + let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD"); + assert!(lower.is_some()); + assert_eq!(lower, upper); + } + + #[test] + fn decode_infohash_accepts_base32() { + let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U"); + assert!(raw.is_some()); + } + + #[test] + fn decode_infohash_rejects_wrong_shape() { + assert!(decode_infohash("tooshort").is_none()); + assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none()); + assert!(decode_infohash("").is_none()); + } +} diff --git a/crates/meta-feeder-sdk/src/config.rs b/crates/meta-feeder-sdk/src/config.rs new file mode 100644 index 0000000..c4af5d6 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config.rs @@ -0,0 +1,312 @@ +//! Self-describing per-plugin configuration: schema, redaction, merge, and the +//! generic schema-driven HTML page the feeder serves at `/config`. +//! +//! The point of this module is that **the gateway carries no per-plugin config +//! knowledge** and **the SDK carries no per-plugin field names**. A plugin +//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema); +//! everything here (redaction of secrets, merge-on-save, and the rendered form) +//! is driven entirely by that schema. Add a feeder → it shows up with a working +//! config form and zero UI/gateway changes. +//! +//! ## Persistence & precedence +//! +//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to +//! `configure()`). The plugin reads **file-or-env** at startup: env is the +//! first-boot seed and stays authoritative until the operator saves through the +//! UI, at which point `config.json` exists and wins. There is no runtime reload +//! — a save returns `restart_required: true` (mirrors the gateway's +//! `gateway-config.json` posture). +//! +//! ## Secrets +//! +//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to +//! the page (redacted to a `_set: bool`), and a blank incoming value on +//! save means "keep the stored secret". + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A plugin's full config surface — an ordered list of fields. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ConfigSchema { + pub fields: Vec, +} + +impl ConfigSchema { + /// Convenience: `true` when the plugin declares no configurable fields (the + /// page then renders a "no configuration required" notice). + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } +} + +/// One configurable field. `record_array` fields nest `fields` for the +/// per-record sub-schema (e.g. torznab's `indexers`). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ConfigField { + /// JSON key in the config object. + pub key: String, + /// Human label for the form. + pub label: String, + #[serde(rename = "type")] + pub kind: FieldKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub help: Option, + #[serde(default)] + pub required: bool, + /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated + /// as the record identifier for secret back-fill on merge (e.g. `name`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +impl ConfigField { + pub fn text(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Text) + } + pub fn secret(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Secret) + } + pub fn bool(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Bool) + } + pub fn number(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Number) + } + pub fn list(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::List) + } + pub fn record_array(key: &str, label: &str, fields: Vec) -> Self { + let mut f = Self::new(key, label, FieldKind::RecordArray); + f.fields = Some(fields); + f + } + fn new(key: &str, label: &str, kind: FieldKind) -> Self { + Self { + key: key.to_string(), + label: label.to_string(), + kind, + help: None, + required: false, + fields: None, + } + } + pub fn with_help(mut self, help: &str) -> Self { + self.help = Some(help.to_string()); + self + } + pub fn required(mut self) -> Self { + self.required = true; + self + } +} + +/// Field widget / value type. Drives both rendering and the secret/merge rules. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FieldKind { + /// Free text → JSON string. + Text, + /// Write-only secret → JSON string; redacted on read, "blank keeps" on save. + Secret, + /// Checkbox → JSON bool. + Bool, + /// Numeric input → JSON number. + Number, + /// Comma/line list → JSON array of strings. + List, + /// Repeatable record → JSON array of objects keyed by nested `fields`. + RecordArray, +} + +fn is_blank(v: Option<&Value>) -> bool { + match v { + None | Some(Value::Null) => true, + Some(Value::String(s)) => s.is_empty(), + _ => false, + } +} + +/// Produce a UI-safe view of `values`: secret fields are dropped and replaced +/// with a `_set` boolean; everything else passes through. Recurses into +/// `record_array` records for secret sub-fields. +pub fn redact(values: &Value, schema: &ConfigSchema) -> Value { + let src = values.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&src, &schema.fields)) +} + +fn redact_object(src: &Map, fields: &[ConfigField]) -> Map { + let mut out = Map::new(); + for f in fields { + match f.kind { + FieldKind::Secret => { + let set = !is_blank(src.get(&f.key)); + out.insert(format!("{}_set", f.key), Value::Bool(set)); + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let arr = src + .get(&f.key) + .and_then(Value::as_array) + .map(|recs| { + recs.iter() + .map(|r| { + let m = r.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&m, subs)) + }) + .collect::>() + }) + .unwrap_or_default(); + out.insert(f.key.clone(), Value::Array(arr)); + } + _ => { + if let Some(v) = src.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// Merge `incoming` (from the UI) onto `base` (the stored/effective config), +/// schema-aware. Returns the full, UNREDACTED object to persist: +/// - secret: blank/absent incoming keeps `base`; non-blank overwrites. +/// - text/number/bool/list: present in incoming overwrites; absent keeps base. +/// - record_array: incoming array replaces base, but blank secret sub-fields are +/// back-filled from the base record matched by the first sub-field (identifier). +pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value { + let base_obj = base.as_object().cloned().unwrap_or_default(); + let inc_obj = incoming.as_object().cloned().unwrap_or_default(); + Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields)) +} + +fn merge_object( + base: &Map, + inc: &Map, + fields: &[ConfigField], +) -> Map { + let mut out = base.clone(); + for f in fields { + match f.kind { + FieldKind::Secret => { + if !is_blank(inc.get(&f.key)) { + out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap()); + } + // blank/absent → keep base (no-op) + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let id_key = subs.first().map(|s| s.key.clone()); + let base_recs = base.get(&f.key).and_then(Value::as_array).cloned(); + let inc_recs = match inc.get(&f.key).and_then(Value::as_array) { + Some(r) => r.clone(), + None => continue, // absent → keep base array untouched + }; + let merged: Vec = inc_recs + .iter() + .map(|rec| { + let rec_obj = rec.as_object().cloned().unwrap_or_default(); + // Find the matching base record by identifier sub-field. + let base_match = match (&id_key, base_recs.as_ref()) { + (Some(idk), Some(brs)) => brs + .iter() + .find(|br| br.get(idk) == rec_obj.get(idk)) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(), + _ => Map::new(), + }; + Value::Object(merge_object(&base_match, &rec_obj, subs)) + }) + .collect(); + out.insert(f.key.clone(), Value::Array(merged)); + } + _ => { + if let Some(v) = inc.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// The generic, schema-driven config page served at `GET /config`. Self-contained +/// (inline CSS + JS, no external assets) so it loads identically when hit +/// directly on the feeder OR reverse-proxied through the gateway. All API calls +/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant. +pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html"); + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn torznab_like() -> ConfigSchema { + ConfigSchema { + fields: vec![ + ConfigField::secret("tmdb_token", "TMDB token"), + ConfigField::bool("resolve_torrent_files", "Resolve .torrent"), + ConfigField::record_array( + "indexers", + "Indexers", + vec![ + ConfigField::text("name", "Name"), + ConfigField::secret("api_key", "API key"), + ], + ), + ], + } + } + + #[test] + fn redact_hides_secrets_and_flags_set() { + let v = json!({ + "tmdb_token": "abc", + "resolve_torrent_files": true, + "indexers": [{"name":"prowlarr","api_key":"k"}] + }); + let r = redact(&v, &torznab_like()); + assert_eq!(r["tmdb_token_set"], json!(true)); + assert!(r.get("tmdb_token").is_none()); + assert_eq!(r["resolve_torrent_files"], json!(true)); + assert_eq!(r["indexers"][0]["name"], json!("prowlarr")); + assert_eq!(r["indexers"][0]["api_key_set"], json!(true)); + assert!(r["indexers"][0].get("api_key").is_none()); + } + + #[test] + fn merge_blank_secret_keeps_base() { + let base = json!({"tmdb_token": "keepme"}); + let inc = json!({"tmdb_token": ""}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("keepme")); + } + + #[test] + fn merge_nonblank_secret_overwrites() { + let base = json!({"tmdb_token": "old"}); + let inc = json!({"tmdb_token": "new"}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("new")); + } + + #[test] + fn merge_record_array_backfills_blank_secret_by_identifier() { + let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]}); + // UI re-submits the indexer with a blank api_key → keep the stored one. + let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["indexers"][0]["api_key"], json!("secret")); + assert_eq!(m["indexers"][0]["name"], json!("prowlarr")); + } + + #[test] + fn merge_bool_overwrites_when_present() { + let base = json!({"resolve_torrent_files": true}); + let inc = json!({"resolve_torrent_files": false}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["resolve_torrent_files"], json!(false)); + } +} diff --git a/crates/meta-feeder-sdk/src/config_page.html b/crates/meta-feeder-sdk/src/config_page.html new file mode 100644 index 0000000..84d6ae2 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config_page.html @@ -0,0 +1,245 @@ + + + + + +feeder config + + + +

Plugin configuration

+

Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect.

+ +
+ + + + + diff --git a/crates/meta-feeder-sdk/src/enrich.rs b/crates/meta-feeder-sdk/src/enrich.rs new file mode 100644 index 0000000..c7315c4 --- /dev/null +++ b/crates/meta-feeder-sdk/src/enrich.rs @@ -0,0 +1,590 @@ +//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline. +//! +//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base +//! record, then drives the **reused meta-sort enrichment plugins** +//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each +//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to +//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches +//! enrichment — full separation of concerns. +//! +//! ## Ordering — why we poll +//! +//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from +//! the record), and those are produced by filename-parser. So the pipeline is +//! **filename-parser → tmdb**. The plugin `/process` contract is *async* +//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after +//! kicking filename-parser we **poll meta-core** until `originalTitle` appears +//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps +//! the feeder stateless — no callback-correlation bookkeeping. The plugins' +//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`. +//! +//! The whole pipeline runs in a spawned background task so it never adds latency +//! to the libp2p resolve path; the meta-share ingester propagates the enriched +//! record on its next poll (~eventually consistent). + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use tracing::{debug, warn}; + +use crate::meta_core::{self, Provenance}; +use crate::types::DiscoveryRecord; + +/// Operator-supplied enrichment wiring, parsed once from the feeder env. +#[derive(Clone, Debug)] +pub struct EnrichmentConfig { + /// meta-core root the feeder self-publishes to + hands to the plugins. + pub meta_core_url: String, + /// `metamesh-plugin-filename-parser` base URL (optional). + pub filename_parser_url: Option, + /// `metamesh-plugin-tmdb` base URL (optional). + pub tmdb_url: Option, + /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`. + pub tmdb_token: Option, + /// TMDB metadata language (default `en-US`). + pub tmdb_language: String, + /// Stable per-deployment id stamped into `provenance.gatewayPeer`. + pub gateway_peer: String, + /// URL the plugins POST their completion callback to (the feeder's own + /// no-op `/enrich/callback`). Polling is the real completion signal; this + /// only keeps the plugins from logging a failed-callback warning. + pub callback_url: String, + /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the + /// pipeline drives subtitle fetching after tmdb resolves the ids — the + /// single reference implementation of OpenSubtitles fetching (the gateway + /// no longer carries its own). + pub opensubtitles_url: Option, + /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`. + pub opensubtitles_api_key: Option, + /// OpenSubtitles account username (required by the plugin for downloads). + pub opensubtitles_username: Option, + /// OpenSubtitles account password. + pub opensubtitles_password: Option, + /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). + pub opensubtitles_languages: String, +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +impl EnrichmentConfig { + /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset — + /// the feeder then falls back to the legacy "return record, gateway stores" + /// path (backward compatible; used by tests and gateway-stores deployments). + pub fn from_env() -> Option { + let meta_core_url = env_nonempty("META_CORE_URL")?; + Some(EnrichmentConfig { + meta_core_url, + filename_parser_url: env_nonempty("FILENAME_PARSER_URL"), + tmdb_url: env_nonempty("TMDB_PLUGIN_URL"), + tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("TORZNAB_TMDB_TOKEN")), + tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()), + gateway_peer: env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()), + callback_url: env_nonempty("META_FEEDER_CALLBACK_URL") + .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()), + opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"), + opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"), + opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"), + opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), + opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") + .unwrap_or_else(|| "en".to_string()), + }) + } +} + +/// How long to wait for filename-parser to merge `originalTitle` before handing +/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous +/// ceiling, not an expected wait. +const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5); +/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the +/// sibling files. tmdb hits the network, so a longer ceiling than filename. +const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Show-level TMDB fields that are **fanned** from the primary file to every +/// sibling file in a multi-file (season-pack) torrent — they're identical for +/// every episode of a show, so we resolve tmdb **once** and copy these across +/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which +/// gateway BT records don't carry, so without this each file would re-hit TMDB). +/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`, +/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file. +const ANCHOR_EXACT: &[&str] = &[ + "tmdbid", + "imdbid", + "poster", + "backdrop", + "rating", + "releasedate", + "studio", + "genres", + "tags", +]; +const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"]; + +/// One file (or pack) to enrich: its CID, the filename to parse, and the base +/// fields already written to meta-core (handed to the plugins as `existingMeta`). +#[derive(Clone, Debug)] +pub struct EnrichTarget { + pub cid: String, + pub file_name: String, + pub base: BTreeMap, +} + +/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to +/// clone (just a reqwest client + an `Arc`), so it can be moved into a +/// spawned per-resolve task. +#[derive(Clone)] +pub struct Enricher { + http: reqwest::Client, + cfg: Arc, +} + +#[derive(Serialize)] +struct ProcessRequest<'a> { + #[serde(rename = "taskId")] + task_id: String, + cid: &'a str, + #[serde(rename = "filePath")] + file_path: &'a str, + #[serde(rename = "callbackUrl")] + callback_url: &'a str, + #[serde(rename = "metaCoreUrl")] + meta_core_url: &'a str, + #[serde(rename = "existingMeta")] + existing_meta: &'a BTreeMap, +} + +impl Enricher { + pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self { + Enricher { + http, + cfg: Arc::new(cfg), + } + } + + pub fn meta_core_url(&self) -> &str { + &self.cfg.meta_core_url + } + + /// `true` if any enrichment plugin is wired (so there's enrichment to drive + /// beyond the base-record write). + pub fn has_plugins(&self) -> bool { + self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some() + } + + /// Write the feeder's base record into meta-core (merge/upsert) under `cid`. + /// Identical shape to the gateway dispatcher's write (provenance + the + /// `cids/` key-set collapse). + pub async fn write_base_record( + &self, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now( + &record.upstream_id, + &record.record_id, + &self.cfg.gateway_peer, + ); + let body = meta_core::build_metadata_body(record, &prov); + meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await + } + + /// Run the video enrichment pipeline for one resolved CID: filename-parser + /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each + /// plugin writes its own fields to meta-core. Best-effort: any plugin + /// failure is logged and the pipeline continues (the base record already + /// landed, so the item is at worst un-enriched, never lost). + pub async fn enrich_video( + &self, + cid: &str, + file_name: &str, + base_fields: BTreeMap, + ) { + let mut meta = base_fields; + + // 1. filename-parser → originalTitle / movieYear / videoType / season / episode. + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await { + Ok(()) => { + // Poll for the merged fields so tmdb gets the clean title. + if let Some(merged) = self + .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + { + meta = merged; + } else { + debug!( + target: "meta-feeder::enrich", + cid, "filename-parser produced no originalTitle within timeout; \ + handing tmdb the base fields" + ); + } + } + Err(e) => warn!( + target: "meta-feeder::enrich", + cid, error = %e, "filename-parser /process failed (non-fatal)" + ), + } + } + + // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video + + // a clean originalTitle, both now present on `meta`). + if let Some(tmdb_url) = self.cfg.tmdb_url.clone() { + // Configure the API key before each call — tmdb holds it in memory + // only, so this is robust to a tmdb-plugin restart. Cheap (local). + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /process failed (non-fatal)" + ); + } + } + + // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which + // tmdb writes async). Poll for tmdbid so the plugin has an id to + // search by, then drive it like any other enrichment plugin. + if self.cfg.opensubtitles_url.is_some() { + let merged = self + .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or(meta); + self.enrich_subtitles(cid, file_name, &merged).await; + } + } + + /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files + + /// an optional whole-torrent pack record. filename-parser runs on every + /// target; tmdb runs **once** on the primary video file and its show-level + /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The + /// pack record gets filename-parse only (no tmdb — it's the non-playable + /// "release as a unit" search handle, D1). Best-effort throughout. + pub async fn enrich_bundle(&self, videos: Vec, pack: Option) { + // Pack: filename-parse only, no tmdb. + if let Some(p) = pack { + self.fire_filename(&p).await; + } + + if videos.is_empty() { + return; + } + if videos.len() == 1 { + // Degenerate single-video bundle → the plain pipeline (filename → tmdb). + let v = &videos[0]; + self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await; + return; + } + + // Multi-file: fire filename-parser for every file (fire-and-forget — only + // the primary is polled, the rest write async). + for v in &videos { + self.fire_filename(v).await; + } + + // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen). + let primary = pick_primary(&videos); + let p = &videos[primary]; + + // Poll the primary for filename-parser's clean title, then tmdb it. + let merged = self + .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + .unwrap_or_else(|| p.base.clone()); + + let Some(tmdb_url) = self.cfg.tmdb_url.clone() else { + return; // no tmdb wired → filename-only enrichment, done. + }; + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed"); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed"); + return; + } + + // Read the resolved anchor back off the primary, then fan it to siblings. + let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await + else { + debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan"); + return; + }; + let anchor = extract_anchor(&anchor_rec); + if anchor.is_empty() { + return; + } + for (i, v) in videos.iter().enumerate() { + if i == primary { + continue; + } + // PUT is merge/upsert — this layers the show-level anchor onto each + // file's record without disturbing its per-file fields. + if let Err(e) = + meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await + { + warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); + } + } + + // Subtitles per video: each episode needs its own (its season/episode + // differs), but they share the fanned tmdbid. Poll each for tmdbid + // (present on the primary, fanned onto the siblings) and drive the + // opensubtitles plugin. Best-effort + quota-guarded inside the plugin. + if self.cfg.opensubtitles_url.is_some() { + for v in &videos { + let merged = self + .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or_else(|| v.base.clone()); + self.enrich_subtitles(&v.cid, &v.file_name, &merged).await; + } + } + } + + /// Fire filename-parser `/process` for one target (fire-and-forget; the + /// plugin merges its fields to meta-core async). + async fn fire_filename(&self, t: &EnrichTarget) { + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + if let Err(e) = self + .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base) + .await + { + warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed"); + } + } + } + + /// POST `/process` to a plugin. Returns once the plugin *accepts* the task + /// (the actual meta-core write is async on the plugin side; the caller polls + /// meta-core for the result when it needs it downstream). + async fn process( + &self, + plugin_url: &str, + plugin: &str, + cid: &str, + file_path: &str, + existing_meta: &BTreeMap, + ) -> anyhow::Result<()> { + let url = format!("{}/process", plugin_url.trim_end_matches('/')); + let body = ProcessRequest { + task_id: format!("{plugin}-{cid}-{}", now_nanos()), + cid, + file_path, + callback_url: &self.cfg.callback_url, + meta_core_url: &self.cfg.meta_core_url, + existing_meta, + }; + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("{plugin} POST {url} returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the tmdb plugin with the API key + language. + async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> { + let Some(token) = self.cfg.tmdb_token.clone() else { + anyhow::bail!("no TMDB token configured"); + }; + let url = format!("{}/configure", tmdb_url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { "apiKey": token, "language": self.cfg.tmdb_language } + }); + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("tmdb /configure returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the opensubtitles plugin with the API key, account + /// credentials, and wanted languages. + async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> { + let Some(key) = self.cfg.opensubtitles_api_key.clone() else { + anyhow::bail!("no OpenSubtitles API key configured"); + }; + let cfg_url = format!("{}/configure", url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { + "apiKey": key, + "username": self.cfg.opensubtitles_username, + "password": self.cfg.opensubtitles_password, + "languages": self.cfg.opensubtitles_languages, + } + }); + let resp = self.http.post(&cfg_url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("opensubtitles /configure returned {status}: {text}"); + } + Ok(()) + } + + /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or + /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta` + /// and writes the `subtitles//` key-set straight to meta-core). + /// Best-effort: any failure is logged and swallowed. + async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) { + let Some(url) = self.cfg.opensubtitles_url.clone() else { + return; + }; + if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") { + debug!( + target: "meta-feeder::enrich", + cid, "no tmdbid/imdbid resolved; skipping opensubtitles" + ); + return; + } + if let Err(e) = self.configure_opensubtitles(&url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /process failed (non-fatal)" + ); + } + } + + /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout` + /// elapses; returns the merged field map on success. + async fn poll_until_field( + &self, + cid: &str, + field: &str, + timeout: Duration, + ) -> Option> { + let deadline = SystemTime::now() + timeout; + loop { + match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await { + Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => { + return Some(fields); + } + Ok(_) => {} + Err(e) => debug!( + target: "meta-feeder::enrich", + cid, error = %e, "poll get_record failed" + ), + } + if SystemTime::now() >= deadline { + return None; + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } +} + +fn now_nanos() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// Pick the primary video file index: largest by `sizeByte`, ties broken toward +/// the first (lowest index). Falls back to 0 for an all-unsized bundle. +fn pick_primary(videos: &[EnrichTarget]) -> usize { + videos + .iter() + .enumerate() + .max_by_key(|(i, v)| { + let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0); + // larger size wins; on a tie the smaller index wins (Reverse). + (size, std::cmp::Reverse(*i)) + }) + .map(|(i, _)| i) + .unwrap_or(0) +} + +/// Project a resolved record down to the show-level anchor fields that are safe +/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]). +fn extract_anchor(fields: &BTreeMap) -> BTreeMap { + fields + .iter() + .filter(|(k, _)| { + ANCHOR_EXACT.contains(&k.as_str()) + || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p)) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_env_requires_meta_core_url() { + // Not set in the test env → None (legacy gateway-stores path). + // (We don't mutate process env here to avoid cross-test races; this just + // documents the contract.) + assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some()); + } + + fn tgt(cid: &str, size: u64) -> EnrichTarget { + EnrichTarget { + cid: cid.into(), + file_name: cid.into(), + base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]), + } + } + + #[test] + fn pick_primary_is_largest_then_first() { + let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)]; + // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins. + assert_eq!(pick_primary(&v), 1); + } + + #[test] + fn pick_primary_all_unsized_defaults_first() { + let v = vec![ + EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() }, + EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() }, + ]; + assert_eq!(pick_primary(&v), 0); + } + + #[test] + fn extract_anchor_keeps_show_level_drops_per_file() { + let rec = BTreeMap::from_iter([ + ("tmdbid".to_string(), "10378".to_string()), + ("imdbid".to_string(), "tt1254207".to_string()), + ("poster".to_string(), "bafkPOSTER".to_string()), + ("genres".to_string(), "Animation|Comedy".to_string()), + ("plot/eng".to_string(), "a plot".to_string()), + ("tags".to_string(), "tmdb-verified".to_string()), + // per-file — must NOT be fanned: + ("originalTitle".to_string(), "Big Buck Bunny".to_string()), + ("season".to_string(), "1".to_string()), + ("episode".to_string(), "3".to_string()), + ("fileName".to_string(), "ep3.mkv".to_string()), + ]); + let a = extract_anchor(&rec); + assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng")); + assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid")); + assert!(!a.contains_key("originalTitle"), "originalTitle is per-file"); + assert!(!a.contains_key("season") && !a.contains_key("episode")); + assert!(!a.contains_key("fileName")); + } +} diff --git a/crates/meta-feeder-sdk/src/filename_meta.rs b/crates/meta-feeder-sdk/src/filename_meta.rs new file mode 100644 index 0000000..ada6f27 --- /dev/null +++ b/crates/meta-feeder-sdk/src/filename_meta.rs @@ -0,0 +1,1305 @@ +//! Season / episode extraction from a torrent release title. +//! +//! This is a Rust port of `@metazla/filename-tools` +//! (`packages/filename-tool/src/lib/...`) — specifically the +//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` → +//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the +//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the +//! best season/episode heuristic we have (the meta-sort `torrent` plugin +//! already uses it), so rather than invent a second, weaker heuristic for +//! the gateway we mirror its pattern set and ordering verbatim. +//! +//! ## Differences from the TS original (deliberate, documented) +//! +//! 1. **No parent-folder fallback.** The TS extractor also reads +//! `Season N` out of the *parent directory* name. A torznab hit is a +//! single release title with no directory context, so that branch is +//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup +//! simply has nothing to read. +//! 2. **Rightmost-number fallback uses the captured digits, not the raw +//! regex match.** The TS `extractRightmostNumber` does +//! `parseFloat(fullMatch)` where the match can include a leading `.` +//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat` +//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the +//! bare number) instead, which is what the function *means* to return. +//! +//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so +//! a movie's release year can't be misread as an episode number by the +//! rightmost-number fallback. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; + +/// Structured season/episode extracted from a release title. Each field +/// is the stringified number exactly as the TS extractor would emit it +/// (`"" + n`): whole numbers render without a decimal, half-episode +/// specials keep the `.5` (e.g. `"13"`, `"13.5"`). +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SeasonEpisode { + pub season: Option, + pub episode: Option, + /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`). + /// Set together with [`episode_end`]; mutually exclusive with the + /// single-episode [`episode`] field. + pub episode_start: Option, + /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`). + pub episode_end: Option, + /// `season * 10000 + episode` disambiguator (matches the TS + /// `increment` field). Computed for completeness; the torznab caller + /// currently only persists `season`/`episode`. + pub increment: Option, + /// True when this release is a **pack/batch** rather than a single + /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season + /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with + /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack + /// is deliberately **not** given a single `episode`. + pub is_pack: bool, + /// True when [`season`] came from an explicit season-bearing token + /// (`S01E05`, `Season 2`, `S03`), as opposed to the "episode with no + /// season → assume 1" default. Lets a per-file reparse know whether to + /// borrow the season from the pack/parent title (see + /// [`extract_season_episode_with_context`]). + pub season_explicit: bool, +} + +impl SeasonEpisode { + /// A pack carrying a concrete inclusive episode range. + pub fn has_episode_range(&self) -> bool { + self.episode_start.is_some() && self.episode_end.is_some() + } +} + +// --------------------------------------------------------------------------- +// Pattern fragments — copied 1:1 from +// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`. +// --------------------------------------------------------------------------- + +const SW: &str = r"(?:\.|\b)"; // a dot or word boundary +const S: &str = r"(?:\.|\b|\s)"; +const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored +const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5) +const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5) +const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5) +const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99 +const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5) + +/// Season-AND-episode patterns, longest/most-specific first. Each has +/// exactly two capture groups: group 1 = season, group 2 = episode. +/// Order matters — the first to match wins (mirrors the TS array order). +static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"), + format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"), + format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"), + format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"), + format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season+episode pattern compiles")) + .collect() +}); + +/// Episode-only patterns. Each has exactly one capture group: the episode. +static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"), + format!(r"(?i){SW}E{S}*{EP4MY}{END}"), + format!(r"(?i){SW}Ep{EP4MY}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("episode-only pattern compiles")) + .collect() +}); + +/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number. +static SOLO_EP: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles")); + +// --------------------------------------------------------------------------- +// Episode-range / pack detection. Anime + TV batches encode a *range* of +// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season +// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so +// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a +// batch collapses to one phantom episode. We detect ranges FIRST (before +// `find_season_and_episode`) and emit `episode_start`/`episode_end` + +// `is_pack`, so the caller can stamp `contentKind=pack` instead. +// +// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`: +// - ranges use a `-`/`~` separator with TWO numbers and `end > start`; +// - the bare form requires the start digit to be preceded by a space/`_`/`-` +// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read +// as a range start — that release falls through to the single-episode path; +// - 4-digit year-like / out-of-range operands are rejected. +// --------------------------------------------------------------------------- + +/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`. +static RANGE_BRACKET: LazyLock = LazyLock::new(|| { + Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles") +}); + +/// CJK episode range: `第29-38話` / `第29~38話`. +static RANGE_CJK: LazyLock = LazyLock::new(|| { + Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles") +}); + +/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces +/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a +/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight +/// separator is the disambiguator from the very common `Show 2 - 11` / +/// `Show - 11` episode-separator convention (spaced dash → single episode, not +/// a range); spaced ranges only come through the unambiguous bracketed form. +static RANGE_BARE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)") + .expect("bare range compiles") +}); + +/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single +/// season. Matched only as a fallback when no episode range is present. +static SEASON_RANGE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles") +}); + +/// `(start, end)` is a plausible inclusive episode range: ascending, and +/// neither operand is a 4-digit year / resolution-like number. +fn plausible_range(start: u32, end: u32) -> bool { + end > start && (1..1900).contains(&start) && end < 1900 +} + +/// Detect an episode range anywhere in the (precleaned) title. Returns the +/// first plausible `(start, end)` found, trying the unambiguous bracketed and +/// CJK forms before the bare form. +fn detect_episode_range(s: &str) -> Option<(u32, u32)> { + for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] { + for caps in re.captures_iter(s) { + let start = caps.get(1).and_then(|m| m.as_str().parse::().ok()); + let end = caps.get(2).and_then(|m| m.as_str().parse::().ok()); + if let (Some(a), Some(b)) = (start, end) { + if plausible_range(a, b) { + return Some((a, b)); + } + } + } + } + None +} + +/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix. +static SFV: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles")); + +// --------------------------------------------------------------------------- +// Keyword pre-clean — copied from +// `filename-tool/src/lib/config/KeywordsArray.ts`. +// Removed before season/episode detection so release tags like `x264`, +// `1080p`, `720` can't be mistaken for episode numbers by the fallback. +// --------------------------------------------------------------------------- + +/// Word-bounded keywords (resolution/codec/source/group tags). Order is +/// irrelevant once compiled individually, but we sort longest-first to +/// match the TS constructor (`keyword.length` desc) for identical +/// overlap behaviour. +const KEYWORDS: &[&str] = &[ + "VF", + "aXXo", + "STFR", + "SUBFORCED", + "BrRipx264", + "YIFY", + "TRUEFRENCH", + "VFF", + "ATVP", + "RenewalEX", + "BD", + "DD", + "8bits", + "8 bits", + "10bits", + "10 bits", + "part 1", + "part 2", + "part 3", + "part 4", + "part A", + "part B", + "part C", + "part D", + "AAC5", + "AAC5.1", + "1080i", + "1080p", + "2160p", + "480i", + "480p", + "4k", + "576i", + "576p", + "720", + "720i", + "720p", + "aac", + "aac4", + "ac3", + "amzn", + "apple tv+", + "avi", + "bbc", + "bd5", + "bdrip", + "blu-ray", + "bluray", + "brrip", + "cam", + "cw", + "dc", + "dcu", + "ddp5.1", + "director's cut", + "disney+", + "divx", + "divx5", + "dl", + "dsr", + "dsrip", + "dts", + "dual audio", + "dubbed", + "dvd", + "dvdivx", + "dvdr", + "dvdrip", + "dvdscr", + "dvdscreener", + "eng", + "eng sub", + "esp", + "fan edit", + "fhd", + "flv", + "fs", + "ger", + "german", + "h.264", + "h.265", + "h264", + "h265", + "hardcoded", + "hbo", + "hd", + "hddvd", + "hdr", + "hdrip", + "hdtv", + "hdtvrip", + "hevc", + "hq", + "hrhd", + "hrhdtv", + "hulu", + "imax", + "ita", + "jpn", + "ld", + "md", + "mkv", + "mp3", + "mp4", + "mpeg", + "mpg", + "mq", + "multi", + "multisubs", + "netflix", + "nf", + "nfofix", + "ntsc", + "ogg", + "ogm", + "ova", + "pal", + "pdtv", + "r3", + "r5", + "rerip", + "rsvcd", + "screener", + "sd", + "se", + "subbed", + "svcd", + "tc", + "telecine", + "telesync", + "ts", + "tv series", + "uhd", + "uhdtv", + "uhdv", + "v2", + "vcd", + "vostfr", + "web", + "web-dl", + "webcast", + "webrip", + "wmv", + "ws", + "www", + "x264", + "x265", + "xsvcd", + "xvid", + "xvidvd", + "xxx", + "bits", + "AnimeServ", +]; + +/// Substrings removed without word boundaries (the TS `substringArray`). +const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"]; + +/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first. +static KEYWORD_RES: LazyLock> = LazyLock::new(|| { + let mut kws: Vec<&str> = KEYWORDS.to_vec(); + kws.sort_by_key(|k| std::cmp::Reverse(k.len())); + kws.into_iter() + .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles")) + .collect() +}); + +/// Compiled substring regexes (no boundaries, case-insensitive), longest-first. +static SUBSTRING_RES: LazyLock> = LazyLock::new(|| { + let mut subs: Vec<&str> = SUBSTRINGS.to_vec(); + subs.sort_by_key(|s| std::cmp::Reverse(s.len())); + subs.into_iter() + .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles")) + .collect() +}); + +// --------------------------------------------------------------------------- +// Language tags — anime/torznab upstreams encode language as release-name +// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a +// structured field. `extract_languages` reads *every* tag off the raw title +// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`. +// The gateway writes each as a `languages/ = "true"` key-set member +// (METADATA_KEYS §9) that the preferred-language filter matches against. +// +// Patterns are ported from the battle-tested torrent-name parsers +// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead +// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`, +// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex` +// crate has no lookaround. Two defensive choices do most of the work: +// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They +// collide with ordinary words, names, and domain TLDs. We rely on the +// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead. +// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside +// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`. +// --------------------------------------------------------------------------- + +/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD +/// or path token can't be read as a language. Only `www.`-prefixed runs are +/// removed — a generic `host.tld` matcher would eat dotted release names like +/// `Spy.x.Family`. Runs on the raw (still-dotted) title. +static URL_STRIP: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles")); + +/// Guard phrases removed from the separator-normalized title before language +/// matching, because each contains a substring that would otherwise +/// false-positive a tag. None of these phrases is itself a language tag, so +/// removal is loss-free for detection — this is how we reproduce the upstream +/// parsers' lookbehind guards without lookaround: +/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese) +/// - `tel aviv` → guards `\btel\b` (Telugu) +/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker) +static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b") + .expect("lang guard re compiles") +}); + +/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match +/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js` +/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`), +/// because meta-share's "languages I read" preference is the source of truth +/// the meta-watch filter compares against — the enriched value must be in the +/// same space the chips use. (This deliberately diverges from `gutenberg.rs` / +/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.) +/// +/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for +/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release +/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul` +/// and `fre`: every pattern that matches contributes its code, so the result +/// is a set, not a first-wins pick. +static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| { + [ + // Multi / dual-audio → "mul". `dl` matches only after "web dl" has + // been stripped (see LANG_GUARD_STRIP). + ( + r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b", + "mul", + ), + // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…). + ( + r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b", + "fre", + ), + (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"), + (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"), + (r"\bitalian\b|\bita\b", "ita"), + (r"\bgerman\b|\bger\b|\bdeu\b", "ger"), + ( + r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b", + "spa", + ), + (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"), + (r"\bkorean\b|\bkor\b", "kor"), + ( + r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b", + "chi", + ), + (r"\brussian\b|\brus\b", "rus"), + (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"), + (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"), + (r"\bczech\b|\bcze\b|\bces\b", "cze"), + (r"\bslovak\b|\bslo\b|\bslk\b", "slo"), + (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"), + ( + r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b", + "rum", + ), + (r"\bgreek\b|\bgre\b|\bell\b", "gre"), + (r"\bukrainian\b|\bukr\b", "ukr"), + (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"), + (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"), + (r"\bnorwegian\b|\bnorsk\b", "nor"), + (r"\bdanish\b|\bdansk\b", "dan"), + (r"\bfinnish\b|\bsuomi\b", "fin"), + (r"\bturkish\b|\btur\b", "tur"), + (r"\barabic\b|\bara\b", "ara"), + (r"\bhindi\b|\bhin\b", "hin"), + (r"\btamil\b|\btam\b", "tam"), + (r"\btelugu\b|\btel\b", "tel"), + (r"\bmalayalam\b|\bmal\b", "mal"), + (r"\bvietnamese\b|\bvie\b", "vie"), + (r"\bthai\b|\btha\b", "tha"), + ] + .iter() + .map(|(p, code)| { + ( + Regex::new(&format!("(?i){p}")).expect("language tag re compiles"), + *code, + ) + }) + .collect() +}); + +/// Characters `cleanSpecialChars` replaces with a space (note: `.` and +/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07` +/// structure). +const SPECIAL_CHARS: &[char] = &[ + '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"', + '`', '~', +]; + +/// Extract season/episode from a release title. +/// +/// Faithful to `computeSeasonAndEpisodeFromFilename` → +/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath` +/// (minus the parent-folder branch — see the module doc). +pub fn extract_season_episode(title: &str) -> SeasonEpisode { + let cleaned = preclean(title); + // Range detection runs on the *raw* title (with `~` normalized to `-`), + // not the precleaned one: `preclean` replaces `~` with a space (it's a + // "special char"), which would erase a `26~37` range, and it never touches + // the `(`/`[` brackets the bracketed form anchors on. + let range_src = title.replace('~', "-"); + + // Pack detection runs first — an episode range (`(01-10)`) would otherwise + // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}` + // pattern. A pack is never given a single `episode`. + if let Some((start, end)) = detect_episode_range(&range_src) { + let season = find_pack_season(&cleaned); + return SeasonEpisode { + is_pack: true, + episode_start: Some(start.to_string()), + episode_end: Some(end.to_string()), + // An explicit season token (`S2 (01-10)`) buckets the pack; a bare + // range (`Frieren (01-28)`) leaves the season for the caller. + season: season.map(|s| s.to_string()), + season_explicit: season.is_some(), + ..Default::default() + }; + } + // Season range (`S01-S03`) — a whole-series pack, no single season/episode. + if SEASON_RANGE.is_match(&range_src) { + return SeasonEpisode { + is_pack: true, + ..Default::default() + }; + } + + let (sea, ep) = find_season_and_episode(&cleaned); + + let mut out = SeasonEpisode::default(); + // season/episode 0 are valid; only < 0 means "not found". + let mut sea_eff = sea; + if sea >= 0.0 { + out.season = Some(fmt_num(sea)); + out.season_explicit = true; + } + if ep >= 0.0 { + out.episode = Some(fmt_num(ep)); + if sea_eff < 0.0 { + // An episode with no season — assume season 1 (matches TS). The + // season is *not* explicit, so a per-file reparse may override it + // from the parent (pack) title. + sea_eff = 1.0; + out.season = Some("1".to_string()); + } + out.increment = Some(fmt_num(sea_eff * 10000.0 + ep)); + } else if sea >= 0.0 { + // Season present, no episode → a whole-season pack (`Season 3`, `S03`). + out.is_pack = true; + } + out +} + +/// Extract season/episode using a `/` style context — the +/// per-file path of an opened multi-file (pack) torrent. Parses the file's own +/// `filename` first; when the filename yields an episode but **no** season, the +/// season is recovered from the `parent` (the pack/torrent title). This +/// restores the parent-folder season fallback the TS +/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped +/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves +/// to S2E3 rather than the inherited pack-level number. +pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode { + let mut file_se = extract_season_episode(filename); + // Only borrow the parent's season for a concrete single episode whose own + // filename carries no *explicit* season token (so its season is the assumed + // `1` default). A per-file record is an episode, not a pack, so we don't + // propagate the parent's pack/range shape here. + if file_se.episode.is_some() && !file_se.season_explicit { + if let Some(s) = find_pack_season(&preclean(parent)) { + file_se.season = Some(s.to_string()); + file_se.season_explicit = true; + if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) { + file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep)); + } + } + } + file_se +} + +/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`, +/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to +/// recover a per-file season from the parent. Excludes the bare-number season +/// form so an episode-only title isn't mis-bucketed. +fn find_pack_season(s: &str) -> Option { + let padded = format!(" {s} "); + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let v = parse_group(&caps, 1); + if v >= 0.0 { + return Some(v as i64); + } + } + } + None +} + +/// Best-effort set of languages a release supports (audio **and** subtitles, +/// mixed), inferred from its title tags. +/// +/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`, +/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we +/// read them off the raw title here. Returns the **full set** of ISO 639-2/B +/// alpha-3 codes found (matching meta-share's UI vocabulary — see +/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is +/// empty when the title carries no recognizable tag — the caller turns that +/// into the `languages/und` sentinel. +pub fn extract_languages(title: &str) -> BTreeSet { + // 1. Strip `www.…` domain noise while the dots are still intact. + let stripped = URL_STRIP.replace_all(title, " "); + // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags + // anchor inside dotted/underscored/hyphenated release names — and so + // the guard phrases ("web dl", "shang chi") match regardless of which + // separator the original used. + let norm: String = stripped + .chars() + .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c }) + .collect(); + // 3. Remove guard phrases that would otherwise false-positive a tag. + let guarded = LANG_GUARD_STRIP.replace_all(&norm, " "); + // 4. Collect every matching code (a set — dedupes, deterministic order). + let mut out = BTreeSet::new(); + for (re, code) in LANGUAGE_TAGS.iter() { + if re.is_match(&guarded) { + out.insert((*code).to_string()); + } + } + out +} + +// --------------------------------------------------------------------------- +// Quality — the resolution *tier* of a release, normalized to a single label +// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured +// resolution, so — like season/episode and language — the release title is the +// only source. Mirrors the client-side `parseVersion()` heuristic in +// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's +// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to +// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height. +// --------------------------------------------------------------------------- + +/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH` +/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title +/// carries no recognizable resolution. +pub fn extract_quality(title: &str) -> Option { + static TIER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles") + }); + static WXH: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles")); + // Normalize `.`/`_` separators so `\b`-anchored tokens match inside + // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`). + let t = title.replace(['.', '_'], " "); + if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) { + let tok = m.as_str().to_lowercase(); + return Some( + match tok.as_str() { + "4k" => "2160p", + "1080i" => "1080p", + "720i" => "720p", + other => other, + } + .to_string(), + ); + } + if let Some(h) = WXH + .captures(&t) + .and_then(|c| c.get(1)) + .and_then(|m| m.as_str().parse::().ok()) + { + let tier = if h >= 2000 { + "2160p" + } else if h >= 1000 { + "1080p" + } else if h >= 700 { + "720p" + } else if h >= 500 { + "576p" + } else { + "480p" + }; + return Some(tier.to_string()); + } + None +} + +/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12 +/// structured search facet). Title-derived / best-effort — same provenance and +/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in +/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens +/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and +/// any stream-derived value. +pub fn extract_codec(title: &str) -> Option { + static CODEC: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b") + .expect("codec re compiles") + }); + // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored + // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`). + let t = title.replace(['.', '_'], " "); + let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase(); + let cleaned = tok.replace(' ', ""); + let norm = match cleaned.as_str() { + "x265" | "h265" | "hevc" => "hevc", + "x264" | "h264" | "avc" => "h264", + other => other, // av1, xvid, divx, vp9 + }; + Some(norm.to_string()) +} + +/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + +/// `cleanSpace`, in that order. +fn preclean(filename: &str) -> String { + let mut s = filename.to_string(); + for re in KEYWORD_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + // removeSFV — first occurrence only (TS uses a non-global replace). + s = SFV.replace(&s, "").into_owned(); + // cleanSpecialChars — replace each special char with a space. + s = s + .chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect(); + // cleanSpace — collapse runs of whitespace and trim. + s.split_whitespace().collect::>().join(" ") +} + +/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0` +/// for "not found". +fn find_season_and_episode(filename: &str) -> (f64, f64) { + // Pad with spaces so word-boundary fragments anchor at the ends + // exactly as the TS code does (`" " + filename + " "`). + let padded = format!(" {filename} "); + + for re in SEASON_EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + let episode = parse_group(&caps, 2); + return (season, episode); + } + } + for re in EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let episode = parse_group(&caps, 1); + return (-1.0, episode); + } + } + // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught + // here — *before* the bare rightmost-number fallback — so a whole-season + // pack reports `(season, no-episode)` instead of the fallback mis-reading + // its season digit as an episode and `extract_season_episode` then + // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03` + // bug). The TS pipeline relied on a parent-folder lookup for this case, + // which the gateway dropped (see module doc). + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + if season >= 0.0 { + return (season, -1.0); + } + } + } + // Last hope: the rightmost bare number. `if (number)` in the TS is + // falsy for 0, so a lone 0 doesn't count as an episode. + if let Some(n) = rightmost_number(&padded) { + if n != 0.0 { + return (-1.0, n); + } + } + (-1.0, -1.0) +} + +fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 { + caps.get(idx) + .and_then(|m| m.as_str().parse::().ok()) + .unwrap_or(-1.0) +} + +/// `extractRightmostNumber` — the last `soloEp` match's number. +fn rightmost_number(s: &str) -> Option { + SOLO_EP + .captures_iter(s) + .filter_map(|c| c.get(1)) + .last() + .and_then(|m| m.as_str().parse::().ok()) +} + +/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64 +/// `Display` already drops the trailing `.0`, so this is a thin wrapper +/// kept for intent. +fn fmt_num(n: f64) -> String { + format!("{n}") +} + +// --------------------------------------------------------------------------- +// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` + +// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`. +// The gateway previously used a separate hand-rolled `clean_torrent_title` +// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and +// season-word/episode-range stripping — so TMDB enrichment failed on many +// release-name shapes. Reuse the already-ported building blocks instead. +// --------------------------------------------------------------------------- + +/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` / +/// `S N` / trailing bare number and everything after it. +static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + format!(r"(?i){SW}{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-only pattern compiles")) + .collect() +}); + +/// Season-only patterns restricted to the **keyword'd** forms (`Season N`, +/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a +/// whole-season *pack* (no episode) before the bare rightmost-number fallback +/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}` +/// entry: that one matches any lone number and would turn an episode-only +/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because +/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`. +static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-word-only pattern compiles")) + .collect() +}); + +/// `cleanTags` — bracketed/parenthesised/braced groups. +static TAG_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles")); +/// `removeTagSeparator` — stray bracket chars. +static TAG_SEP_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles")); +/// `removeYear` — a standalone 19xx/20xx. +static YEAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles")); +/// `removeBeyondHyphen` — capture everything before the first ` - ` +/// (space-delimited) hyphen. +static BEYOND_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles")); +static TRAILING_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles")); + +fn clean_special_chars(s: &str) -> String { + s.chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect() +} + +fn remove_keywords(s: &str) -> String { + let mut out = s.to_string(); + for re in KEYWORD_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + out +} + +fn remove_beyond_hyphen(s: &str) -> String { + if let Some(c) = BEYOND_HYPHEN_RE.captures(s) { + return c + .get(1) + .map(|m| m.as_str().trim().to_string()) + .unwrap_or_default(); + } + TRAILING_HYPHEN_RE.replace(s, "").trim().to_string() +} + +fn remove_season_and_episode(s: &str) -> String { + for re in SEASON_EPISODE_PATTERNS + .iter() + .chain(EPISODE_PATTERNS.iter()) + .chain(SEASON_ONLY_PATTERNS.iter()) + { + if re.is_match(s) { + return re.replace(s, "").trim().to_string(); + } + } + // removeRightmostNumber fallback. + SOLO_EP.replace(s, "").trim().to_string() +} + +/// Produce a clean show/movie title from a raw release name, faithful to +/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB +/// search — NOT a hand-rolled cleaner. +pub fn clean_title(raw: &str) -> String { + let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`) + s = remove_keywords(&s); // 172 release/quality keywords + v1..v5 + s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags + s = remove_beyond_hyphen(&s); // drop everything after the first " - " + s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets + s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear + s = SFV.replace(&s, "").into_owned(); // removeSFV + s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number + s = s.replace('.', " "); // cleanDot + let s = s.trim().trim_start_matches('-').trim_end_matches('-'); + s.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn se(title: &str) -> (Option, Option) { + let r = extract_season_episode(title); + (r.season, r.episode) + } + + #[test] + fn strict_sxxeyy() { + assert_eq!( + se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"), + (Some("1".into()), Some("2".into())) + ); + } + + #[test] + fn dotted_release_name() { + assert_eq!( + se("The.Wire.S03E07.720p.BluRay.x264"), + (Some("3".into()), Some("7".into())) + ); + } + + #[test] + fn clean_title_collapses_release_variants() { + let ct = |s: &str| clean_title(s).to_lowercase(); + // Season-word, underscore/range, dotted+EnN, and the trailing-num + // Nyaa shape all collapse to the bare show title (the cases that + // were failing TMDB enrichment). + assert_eq!( + ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), + "spy x family" + ); + assert_eq!( + ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"), + "spy x family" + ); + assert_eq!( + ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"), + "spy x family" + ); + assert_eq!( + ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"), + "spy x family" + ); + } + + #[test] + fn concise_nxnn() { + assert_eq!( + se("Some Show 1x05 720p x264"), + (Some("1".into()), Some("5".into())) + ); + } + + #[test] + fn bracketed_concise() { + assert_eq!( + se("Some Show [2x10] HDTV"), + (Some("2".into()), Some("10".into())) + ); + } + + #[test] + fn verbose_season_episode() { + assert_eq!( + se("My Show Season 2 Episode 13"), + (Some("2".into()), Some("13".into())) + ); + } + + #[test] + fn anime_trailing_number_assumes_season_one() { + // No season marker → episode via rightmost-number fallback, + // season assumed 1. SFV id stripped, brackets ignored. + assert_eq!( + se("[SubsPlease] Some Anime - 117 [FC412C51]"), + (Some("1".into()), Some("117".into())) + ); + } + + #[test] + fn half_episode_special_is_preserved() { + assert_eq!( + se("Some Anime S01E13.5"), + (Some("1".into()), Some("13.5".into())) + ); + } + + #[test] + fn increment_is_season_times_10000_plus_episode() { + let r = extract_season_episode("Show S02E05"); + assert_eq!(r.increment.as_deref(), Some("20005")); + } + + #[test] + fn four_digit_year_is_not_misread_as_episode() { + // The `soloEp` fallback caps at 3 digits, so a 4-digit release + // year can't be picked up — no episode here. + let r = extract_season_episode("The Matrix 1999 1080p BluRay x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn short_number_in_movie_title_is_grabbed_by_fallback() { + // A bare 1-3 digit number in a movie title DOES trip the + // rightmost-number fallback (here: "13"). This is exactly why the + // torznab caller gates season/episode on TV-ness — a movie record + // must not inherit this bogus episode. + let r = extract_season_episode("Apollo 13 1080p BluRay x264"); + assert_eq!(r.episode.as_deref(), Some("13")); + } + + #[test] + fn keyword_strip_prevents_codec_false_positive() { + // "x264" must be removed so "264" isn't read as the episode. + let r = extract_season_episode("Plain Movie Title x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn no_numbers_yields_nothing() { + let r = extract_season_episode("Just A Plain Title"); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + } + + #[test] + fn season_only_pack_is_not_misread_as_episode() { + // Whole-season packs (no episode) must report the season with NO + // episode — not `S01E0N` from the rightmost-number fallback. This is + // the original bug: `Spy x Family Season 3 (S03)` was parsed as + // season 1, episode 3, dumping the S3 pack into the Season-1 bucket. + assert_eq!( + se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"), + (Some("3".into()), None) + ); + assert_eq!( + se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"), + (Some("2".into()), None) + ); + // Bare `S03` pack (no `Season` word) via the S-prefixed pattern. + assert_eq!( + se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"), + (Some("3".into()), None) + ); + assert_eq!( + se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"), + (Some("3".into()), None) + ); + // French season word. + assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None)); + } + + #[test] + fn explicit_sxxeyy_still_wins_over_season_only() { + // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected + // by the new season-only branch. + assert_eq!( + se("Spy x Family Season 3 S03E13 1080p"), + (Some("3".into()), Some("13".into())) + ); + } + + #[test] + fn episode_range_is_a_pack_not_sxxeyy() { + // The headline bug: `(01-10)` was mis-read as S1 E10 by the + // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no + // single `episode`. + let r = extract_season_episode("Frieren (01-28) [Batch] 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("28")); + + // Bare, dash-delimited, leading-zero range. + let r = extract_season_episode("Some Anime 01-12 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("12")); + + // Tilde range, no leading zero. + let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("26")); + assert_eq!(r.episode_end.as_deref(), Some("37")); + } + + #[test] + fn explicit_season_buckets_a_pack() { + // `S2 (01-10)` → pack in season 2, episodes 1–10. + let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("10")); + } + + #[test] + fn cjk_episode_range_is_a_pack() { + let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("29")); + assert_eq!(r.episode_end.as_deref(), Some("38")); + } + + #[test] + fn season_range_is_a_pack_with_no_single_season() { + let r = extract_season_episode("Frieren S01-S03 Complete 1080p"); + assert!(r.is_pack); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start, None); + } + + #[test] + fn whole_season_release_is_a_pack() { + // A season-only release (`Season 3`, `S03`, no episode) is a pack too — + // the original "season-level release shows as one media card" bug. + let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("3")); + assert_eq!(r.episode, None); + } + + #[test] + fn single_episode_is_not_a_pack() { + // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all + // single episodes, never packs. + for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] { + assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack"); + } + let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("7")); + + // The `Show - ` anime convention (spaced dash) is a + // single episode, NOT a `2..11` range — the tight-separator rule keeps + // this out of pack detection. + let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("11")); + + // A year range in a movie title is not an episode range. + assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack); + } + + #[test] + fn per_file_context_recovers_season_from_parent() { + // Opened pack: the inner file carries the episode, the pack title the + // season. `Frieren - 03.mkv` alone assumes S1; with the S2 pack parent + // it resolves to S2E3. + let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv"); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("3")); + assert!(!r.is_pack); + + // The file's own explicit season wins over the parent. + let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv"); + assert_eq!(r.season.as_deref(), Some("1")); + assert_eq!(r.episode.as_deref(), Some("5")); + } + + #[test] + fn absolute_numbered_anime_still_assumes_season_one() { + // No season keyword → the rightmost-number fallback still fires, season + // assumed 1 (Phase 3 remaps this against TMDB episode counts later). + assert_eq!( + se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"), + (Some("1".into()), Some("37".into())) + ); + } + + fn langs(s: &str) -> BTreeSet { + extract_languages(s) + } + + #[test] + fn language_from_release_tags() { + // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi). + assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre")); + assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger")); + assert!(langs("Show ENG SUB 720p").contains("eng")); + assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn")); + assert!(langs("Film TRUEFRENCH 1080p").contains("fre")); + } + + #[test] + fn multi_is_mul_and_collects_every_language() { + // `languages` is the union of all supported audio+sub languages. + // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is + // ALSO collected (the set, not a first-wins pick). + assert_eq!( + langs("Show MULTI VOSTFR 1080p"), + BTreeSet::from(["mul".to_string(), "fre".to_string()]) + ); + assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul")); + assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul")); + // A bilingual release collects both specific codes. + assert_eq!( + langs("Movie.2021.GERMAN.ENGLISH.1080p"), + BTreeSet::from(["ger".to_string(), "eng".to_string()]) + ); + } + + #[test] + fn no_language_tag_yields_empty_set() { + // A plain quality/codec release name carries no language tag — the + // caller maps the empty set to the `languages/und` sentinel. + assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty()); + // Short codes embedded in words/names must not false-positive: "ita" + // inside "digital", "por" inside "portal", etc. + assert!(langs("VFX Reel 2024 1080p").is_empty()); + assert!(langs("Digital Capital Portal 1080p").is_empty()); + } + + #[test] + fn lookaround_guards_reproduced_in_code() { + // WEB-DL must not read as `dl`→mul (German dual-language marker). + assert!(!langs("Movie 2021 WEB-DL x264").contains("mul")); + // "Shang-Chi" must not read as `chi` (Chinese)… + assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi")); + // …but a real CHI tag still does. + assert!(langs("Movie.2021.CHI.1080p").contains("chi")); + // "Tel Aviv" must not read as `tel` (Telugu). + assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel")); + // www. domain noise (incl. its TLD) must not contribute a language. + assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty()); + } + + #[test] + fn quality_tiers_and_normalization() { + let q = |s: &str| extract_quality(s); + assert_eq!( + q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(), + Some("1080p") + ); + assert_eq!( + q("The.Wire.S03E07.720p.BluRay.x264").as_deref(), + Some("720p") + ); + // 4k / interlaced collapse to the canonical progressive tier. + assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p")); + assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p")); + assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p")); + // WxH fallback, mapped by height; dot/underscore separators handled. + assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p")); + assert_eq!(q("Show 1280x720").as_deref(), Some("720p")); + // No resolution token → None (never guess). + assert_eq!(q("Just A Plain Title"), None); + } + + #[test] + fn codec_normalization() { + let c = |s: &str| extract_codec(s); + // HEVC encoder spellings collapse to the codec name. + assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc")); + assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc")); + assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc")); + // AVC encoder spellings collapse to h264. + assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264")); + assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264")); + // Others kept as-is. + assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1")); + assert_eq!(c("Old Release XviD").as_deref(), Some("xvid")); + // No codec token → None (never guess). + assert_eq!(c("Just A Plain Title 1080p"), None); + } +} diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs new file mode 100644 index 0000000..d8523d3 --- /dev/null +++ b/crates/meta-feeder-sdk/src/hash.rs @@ -0,0 +1,521 @@ +//! Content-addressed identifiers used by feeder plugins' `compute_outcomes` +//! to hash upstream-fetched bytes. +//! +//! Two function families live here: +//! +//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] — +//! the size-prefix-plus-middle-1MB-sample identification primitive shared +//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`. +//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the +//! middle slice directly so plugins that can only fetch a partial range +//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file +//! just to throw most of it away. **Rust-only by construction.** +//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full +//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec +//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches +//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +//! --hash=sha2-256 `. +//! +//! Cross-implementation parity for `compute_midhash256` with the TypeScript +//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression +//! silently breaks dedupe across the platform. + +use bytes::Bytes; +use sha2::{Digest, Sha256}; + +/// Sample window in bytes (1 MiB). Files this size or smaller are hashed +/// in full; larger files are hashed over the middle 1 MiB slice. +const SAMPLE_SIZE: usize = 1024 * 1024; + +/// Per-block IPFS chunk size (256 KiB), matches kubo's +/// `--chunker=size-262144` default. +pub const IPFS_CHUNK_SIZE: usize = 256 * 1024; + +/// dag-pb fanout — children per internal node. Matches kubo's balanced +/// builder default. +pub const IPFS_FANOUT: usize = 174; + +/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every +/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by +/// their own CID. Pass `.blocks` to the gateway's blockstore so peers +/// can fetch the file by CID via bitswap. +#[derive(Debug, Clone)] +pub struct IpfsBlocks { + /// Canonical "bafy…" / "bafk…" root cid string. Identical to what + /// [`compute_ipfs_cid`] returns for the same input. + pub root: String, + /// Every block — root included — keyed by its own cid. For a + /// single-leaf file, this is a one-entry vec `[(root, payload)]` + /// (raw codec, payload is the file bytes). For multi-leaf files, + /// it is `leaves ++ internal_nodes` in build order; the **last** + /// entry is the root. + pub blocks: Vec<(String, Bytes)>, +} + +/// Compute a midhash256 CID matching the TypeScript implementation in +/// `meta-hash/src/lib/file-id/FastHash.ts`: +/// +/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the +/// whole file if it fits in 1 MiB). +/// 2. SHA-256 it. +/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field +/// and the multihash hash-code field, length 32, then multibase-base32-lower +/// with the `b` prefix. +/// +/// **Endianness for the size prefix is big-endian** to match TS's +/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every +/// import — keep the test pinned. +pub fn compute_midhash256(bytes: &[u8]) -> String { + let size = bytes.len(); + let sample: &[u8] = if size <= SAMPLE_SIZE { + bytes + } else { + let start = (size - SAMPLE_SIZE) / 2; + &bytes[start..start + SAMPLE_SIZE] + }; + compute_midhash256_from_sample(size as u64, sample) +} + +/// Same midhash256 primitive as [`compute_midhash256`], but takes the +/// sample bytes directly rather than slicing them out of a full file. +/// For upstreams that can only deliver a middle range (BitTorrent BEP 9, +/// sliced HTTP), this avoids forcing the caller to buffer the whole file +/// in memory just to throw most of it away. +/// +/// `total_size` is the **full file size in bytes** — used as the +/// size prefix, NOT `middle_sample.len()`. +pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String { + let size_be = total_size.to_be_bytes(); + let mut hasher = Sha256::new(); + hasher.update(size_be); + hasher.update(middle_sample); + let digest: [u8; 32] = hasher.finalize().into(); + + // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20]. + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(38); + wire.push(0x01); + wire.extend_from_slice(&[0x80, 0x20]); + wire.extend_from_slice(&[0x80, 0x20]); + wire.push(0x20); + wire.extend_from_slice(&digest); + + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1. +/// +/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so +/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11), +/// len=20, digest=`. +pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String { + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(4 + 20); + wire.push(0x01); // CIDv1 version + wire.push(0x55); // codec: raw + wire.push(0x11); // multihash: sha1 + wire.push(0x14); // multihash length: 20 + wire.extend_from_slice(infohash_20); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a single file inside a **BitTorrent v1** torrent +/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`. +pub const BTIH_V1_FILE_CODEC: u64 = 0x1001; + +/// Custom multicodec for a single file inside a **BitTorrent v2** torrent +/// (`btih-v2-file`). +pub const BTIH_V2_FILE_CODEC: u64 = 0x1002; + +/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent, +/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info` +/// dict) plus the zero-based file index in `info.files`. Single-file +/// torrents use index `0`. +pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String { + bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index) +} + +/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent +/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's +/// canonical traversal index in `file tree`. +pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String { + bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index) +} + +/// Shared encoder for the two torrent-file CID families. `infohash` is 20 +/// bytes for v1 / 32 for v2; `codec` selects the family. +fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String { + // digest = infohash ‖ varint(file_index) + let mut digest = Vec::with_capacity(infohash.len() + 2); + digest.extend_from_slice(infohash); + write_pb_varint(file_index, &mut digest); + + // CIDv1: [version][codec varint][multihash code varint][len varint][digest] + let mut wire = Vec::with_capacity(1 + 4 + digest.len()); + wire.push(0x01); + write_pb_varint(codec, &mut wire); + write_pb_varint(codec, &mut wire); + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's +/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +/// --hash=sha2-256 `. +pub fn compute_ipfs_cid(bytes: &[u8]) -> String { + compute_ipfs_blocks(bytes).root +} + +/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every +/// block produced along the way (leaf chunks + internal dag-pb nodes) +/// so callers can populate a bitswap blockstore in one pass. +pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks { + // Leaves: chunk bytes into raw-codec blocks. Empty input still gets + // one (empty) leaf so the cid is well-defined. + let mut blocks: Vec<(String, Bytes)> = Vec::new(); + let leaves: Vec<(Vec, u64)> = if bytes.is_empty() { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b"")); + blocks.push((cid_string(&cid_wire), Bytes::new())); + vec![(cid_wire, 0)] + } else { + bytes + .chunks(IPFS_CHUNK_SIZE) + .map(|chunk| { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk)); + blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk))); + (cid_wire, chunk.len() as u64) + }) + .collect() + }; + + // Single-leaf file: return the leaf cid as the file cid. + if leaves.len() == 1 { + let root = cid_string(&leaves[0].0); + return IpfsBlocks { root, blocks }; + } + + // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes. + let mut level: Vec<(Vec, u64, u64)> = leaves + .into_iter() + .map(|(cid, size)| (cid, size, size)) + .collect(); + + while level.len() > 1 { + let mut next: Vec<(Vec, u64, u64)> = + Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT)); + for batch in level.chunks(IPFS_FANOUT) { + let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect(); + let filesize: u64 = blocksizes.iter().sum(); + let unixfs_data = encode_unixfs_file(filesize, &blocksizes); + let node_bytes = encode_dagpb_node(batch, &unixfs_data); + let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes)); + blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone()))); + let own_tsize: u64 = + batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64; + next.push((node_cid, filesize, own_tsize)); + } + level = next; + } + let root = cid_string(&level[0].0); + IpfsBlocks { root, blocks } +} + +fn cid_string(wire: &[u8]) -> String { + format!("b{}", base32_lower_no_padding(wire)) +} + +/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code +/// varint=0x12 sha2-256][len=0x20][digest...]`. +fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec { + debug_assert!( + codec < 0x80, + "codec varint must fit in one byte for this helper" + ); + let mut wire = Vec::with_capacity(36); + wire.push(0x01); // CIDv1 + wire.push(codec); + wire.push(0x12); // multihash code: sha2-256 + wire.push(0x20); // digest length: 32 + wire.extend_from_slice(digest); + wire +} + +fn sha2_256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// LEB128 unsigned varint encoding into `out`. +fn write_pb_varint(value: u64, out: &mut Vec) { + let mut v = value; + while v >= 0x80 { + out.push((v as u8) | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) { + write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint) + write_pb_varint(value, out); +} + +fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) { + write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited + write_pb_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +/// UnixFS protobuf payload for a File node: `Type=File, filesize, +/// blocksizes[]`. +fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec { + let mut out = Vec::new(); + write_pb_varint_field(1, 2, &mut out); // Type = File + write_pb_varint_field(3, filesize, &mut out); // filesize + for &bs in blocksizes { + write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated) + } + out +} + +/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order +/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second. +fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec { + let mut out = Vec::new(); + for (cid_wire, _filesize, tsize) in children { + let mut link_bytes = Vec::new(); + write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash + write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize + write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first + } + write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last + out +} + +/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the +/// multibase `b` prefix. +pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut out = String::with_capacity(input.len().div_ceil(5) * 8); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &b in input { + buffer = (buffer << 8) | b as u64; + bits += 8; + while bits >= 5 { + bits -= 5; + let idx = ((buffer >> bits) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + } + if bits > 0 { + let idx = ((buffer << (5 - bits)) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn midhash256_matches_ts_fixtures() { + assert_eq!( + compute_midhash256(b"hello world"), + "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4" + ); + assert_eq!( + compute_midhash256(b""), + "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y" + ); + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_midhash256(&mb_zeros), + "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4" + ); + } + + #[test] + fn midhash256_from_sample_matches_full_bytes() { + let small: &[u8] = b"hello world"; + assert_eq!( + compute_midhash256_from_sample(small.len() as u64, small), + compute_midhash256(small), + ); + + assert_eq!( + compute_midhash256_from_sample(0, b""), + compute_midhash256(b""), + ); + + let one_mb = vec![0u8; SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(one_mb.len() as u64, &one_mb), + compute_midhash256(&one_mb), + ); + + let two_mb = vec![0u8; 2 * SAMPLE_SIZE]; + let start = (two_mb.len() - SAMPLE_SIZE) / 2; + let middle = &two_mb[start..start + SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(two_mb.len() as u64, middle), + compute_midhash256(&two_mb), + ); + } + + #[test] + fn midhash256_from_sample_partial_sample_stable() { + let sample = vec![0u8; SAMPLE_SIZE]; + let total_5gb: u64 = 5 * 1024 * 1024 * 1024; + let total_10gb: u64 = 10 * 1024 * 1024 * 1024; + + let h1 = compute_midhash256_from_sample(total_5gb, &sample); + let h2 = compute_midhash256_from_sample(total_5gb, &sample); + assert_eq!(h1, h2, "same (size, sample) must hash identically"); + + let h3 = compute_midhash256_from_sample(total_10gb, &sample); + assert_ne!( + h1, h3, + "size prefix must contribute to cid — same sample, different size" + ); + } + + #[test] + fn bt_v1_file_cid_wire_layout() { + let infohash: [u8; 20] = [ + 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c, + 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06, + ]; + + let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want.extend_from_slice(&infohash); + want.push(0x04); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 4), + format!("b{}", base32_lower_no_padding(&want)) + ); + + let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want0.extend_from_slice(&infohash); + want0.push(0x00); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want0)) + ); + + let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16]; + want200.extend_from_slice(&infohash); + want200.extend_from_slice(&[0xC8, 0x01]); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 200), + format!("b{}", base32_lower_no_padding(&want200)) + ); + } + + #[test] + fn bt_v2_file_cid_wire_layout() { + let infohash: [u8; 32] = [0xAB; 32]; + let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21]; + want.extend_from_slice(&infohash); + want.push(0x00); + assert_eq!( + compute_bt_v2_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want)) + ); + } + + #[test] + fn bt_file_cid_properties() { + let ih1 = [0x11u8; 20]; + let ih2 = [0x22u8; 32]; + + assert_eq!( + compute_bt_v1_file_cid(&ih1, 3), + compute_bt_v1_file_cid(&ih1, 3) + ); + assert_ne!( + compute_bt_v1_file_cid(&ih1, 0), + compute_bt_v1_file_cid(&ih1, 1) + ); + assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b')); + assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b')); + assert_ne!( + compute_bt_v1_file_cid(&[0xAB; 20], 0), + compute_bt_v2_file_cid(&[0xAB; 32], 0) + ); + } + + #[test] + fn base32_known_vectors() { + assert_eq!(base32_lower_no_padding(b""), ""); + assert_eq!(base32_lower_no_padding(b"f"), "my"); + assert_eq!(base32_lower_no_padding(b"fo"), "mzxq"); + assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6"); + assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq"); + assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi"); + } + + #[test] + fn ipfs_cid_single_leaf_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b"hello world"), + "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" + ); + } + + #[test] + fn ipfs_cid_empty_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b""), + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + } + + #[test] + fn ipfs_cid_multi_leaf_zeros_stable() { + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_ipfs_cid(&mb_zeros), + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + } + + #[test] + fn ipfs_blocks_single_leaf() { + let payload = b"hello world"; + let out = compute_ipfs_blocks(payload); + assert_eq!(out.root, compute_ipfs_cid(payload)); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].0, out.root); + assert_eq!(out.blocks[0].1.as_ref(), payload); + } + + #[test] + fn ipfs_blocks_multi_leaf() { + let mb_zeros = vec![0u8; 1024 * 1024]; + let out = compute_ipfs_blocks(&mb_zeros); + assert_eq!( + out.root, + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + assert_eq!(out.blocks.len(), 5); + assert_eq!(out.blocks.last().unwrap().0, out.root); + for (_cid, block) in &out.blocks[..4] { + assert_eq!(block.len(), 256 * 1024); + assert!(block.iter().all(|&b| b == 0)); + } + } + + #[test] + fn ipfs_blocks_empty() { + let out = compute_ipfs_blocks(b""); + assert_eq!( + out.root, + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].1.len(), 0); + assert_eq!(out.blocks[0].0, out.root); + } +} diff --git a/crates/meta-feeder-sdk/src/lang.rs b/crates/meta-feeder-sdk/src/lang.rs new file mode 100644 index 0000000..3d306c0 --- /dev/null +++ b/crates/meta-feeder-sdk/src/lang.rs @@ -0,0 +1,81 @@ +//! Language-code normalization — ISO 639 alias folding, **not** filename +//! parsing. +//! +//! This lives apart from [`crate::filename_meta`] on purpose: it doesn't read a +//! release title, it canonicalizes a *language code* (an indexer's declared +//! language, a query's `languages:` filter, a title-derived stamp) onto the +//! single ISO 639-2/B alpha-3 vocabulary the rest of the stack compares against. +//! It is the one piece of the old `filename_meta` module that has no equivalent +//! in the `metamesh-plugin-filename-parser` plugin — because it isn't filename +//! parsing — so it stays in Rust while the title-parsing algorithm itself is +//! owned by the plugin. Callers in the **request hot path** (e.g. the torznab +//! indexer-language fan-out skip) need it synchronously, with no network hop. + +/// Normalize a language code to the ISO 639-2/B alpha-3 vocabulary the stack +/// emits (`fre`/`ger`/`chi`, **not** the 639-2/T `fra`/`deu`/`zho` nor the +/// 639-1 two-letter forms), lowercased. This is the space meta-share's +/// "languages I read" preference uses and that the title-derived language stamps +/// write, so an operator who declares an indexer's language as `fra`/`de`/`zh` +/// still lines up with a `fre`/`ger`/`chi` query filter. Unknown / +/// already-canonical codes pass through lowercased+trimmed unchanged. +pub fn normalize_lang_code(code: &str) -> String { + let c = code.trim().to_ascii_lowercase(); + let mapped = match c.as_str() { + // 639-2/T and 639-1 → 639-2/B for the codes that actually differ. + "fra" | "fr" => "fre", + "deu" | "de" => "ger", + "zho" | "zh" => "chi", + "nld" | "nl" => "dut", + "ell" | "el" => "gre", + "ron" | "ro" => "rum", + "slk" | "sk" => "slo", + "ces" | "cs" => "cze", + // 639-1 → 639-2 for the rest of the language-tag output space (these + // share the same /B and /T alpha-3, so only the two-letter form maps). + "en" => "eng", + "ja" => "jpn", + "it" => "ita", + "es" => "spa", + "pt" => "por", + "ko" => "kor", + "ru" => "rus", + "pl" => "pol", + "hu" => "hun", + "uk" => "ukr", + "bg" => "bul", + "sv" => "swe", + "no" => "nor", + "da" => "dan", + "fi" => "fin", + "tr" => "tur", + "ar" => "ara", + "hi" => "hin", + "ta" => "tam", + "te" => "tel", + "ml" => "mal", + "vi" => "vie", + "th" => "tha", + _ => return c, + }; + mapped.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_lang_maps_to_639_2b_vocabulary() { + // 639-2/T and 639-1 spellings collapse onto the /B forms the stack uses. + assert_eq!(normalize_lang_code("fra"), "fre"); + assert_eq!(normalize_lang_code("FR"), "fre"); + assert_eq!(normalize_lang_code("deu"), "ger"); + assert_eq!(normalize_lang_code("zh"), "chi"); + assert_eq!(normalize_lang_code("nld"), "dut"); + // Already-canonical / unknown codes pass through trimmed + lowercased. + assert_eq!(normalize_lang_code(" ENG "), "eng"); + assert_eq!(normalize_lang_code("fre"), "fre"); + assert_eq!(normalize_lang_code("mul"), "mul"); + assert_eq!(normalize_lang_code("xyz"), "xyz"); + } +} diff --git a/crates/meta-feeder-sdk/src/lib.rs b/crates/meta-feeder-sdk/src/lib.rs new file mode 100644 index 0000000..e2c813a --- /dev/null +++ b/crates/meta-feeder-sdk/src/lib.rs @@ -0,0 +1,42 @@ +//! `meta-feeder-sdk` — the source-agnostic foundation shared by the gateway +//! core and every feeder sidecar binary. +//! +//! A *feeder* implements [`FeederPlugin`] (find records, fetch bytes) for one +//! or more upstreams, and [`serve_feeders`] exposes them over the feeder HTTP +//! contract. The gateway core consumes that contract via its +//! `RemoteFeederPlugin` and keeps the libp2p wire, the bitswap blockstore, the +//! hashing-into-the-blockstore, and the meta-core store-back to itself. +//! +//! Deliberately libp2p-free and blockstore-free — see this crate's `Cargo.toml`. + +// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it +// directly rather than boxing on every embedded-DB call. Boxing each would be +// pure noise for a single-process store, so silence the lint crate-wide. +#![allow(clippy::result_large_err)] + +pub mod cache; +pub mod common; +pub mod config; +pub mod enrich; +pub mod filename_meta; +pub mod hash; +pub mod lang; +pub mod meta_core; +pub mod plugin; +pub mod query; +pub mod query_eval; +pub mod serve; +pub mod types; + +pub use config::{ConfigField, ConfigSchema, FieldKind}; +pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher}; +pub use meta_core::FeederStore; +pub use plugin::{ + upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, +}; +pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; +pub use serve::{ + configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, + HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, +}; +pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; diff --git a/crates/meta-feeder-sdk/src/meta_core.rs b/crates/meta-feeder-sdk/src/meta_core.rs new file mode 100644 index 0000000..93cfe52 --- /dev/null +++ b/crates/meta-feeder-sdk/src/meta_core.rs @@ -0,0 +1,293 @@ +//! Lightweight meta-core write/read client for feeders that **self-publish**. +//! +//! Post-"smart feeder / dumb gateway" split: a feeder given a `META_CORE_URL` +//! writes its own base record into meta-core instead of returning it for the +//! gateway dispatcher to store. This keeps the gateway a pure aggregator + +//! request router; the feeder owns the full record lifecycle for its upstream. +//! +//! The write shape is **byte-for-byte the same** as the gateway's +//! `meta_core::build_metadata_body` (`crates/meta-gateway/src/meta_core.rs`): +//! `cid_*` fields collapse to the `cids/ = "true"` key-set, and a +//! `provenance` JSON blob is stamped. Keeping them identical means a record +//! written by the feeder is indistinguishable from one the dispatcher wrote — +//! the meta-share ingester, meta-watch, and ranking all behave the same. +//! +//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core +//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the +//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}` +//! merges accumulate on the same `/file/{cid}` hash. + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use tracing::warn; + +use crate::plugin::{HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, Hash}; + +/// Provenance recorded on every gateway-acquired record. Mirrors the gateway +/// dispatcher's shape so feeder-written and dispatcher-written records are +/// interchangeable. +#[derive(Clone, Debug)] +pub struct Provenance { + pub upstream: String, + pub record_id: String, + /// Stable per-deployment id (the gateway peer this feeder backs, from + /// `META_GATEWAY_PEER_ID`, else the feeder hostname). Analytics group by it. + pub gateway_peer: String, + pub acquired_at: u64, +} + +impl Provenance { + /// Build a provenance stamp for `upstream`/`record_id`, stamping "now". + pub fn now(upstream: &str, record_id: &str, gateway_peer: &str) -> Self { + Provenance { + upstream: upstream.to_string(), + record_id: record_id.to_string(), + gateway_peer: gateway_peer.to_string(), + acquired_at: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + } + } +} + +/// JSON-stringified provenance value. Same key set + `"source":"gateway"` as the +/// gateway dispatcher (`crates/meta-gateway/src/meta_core.rs::provenance_to_json`). +pub fn provenance_to_json(p: &Provenance) -> String { + serde_json::to_string(&serde_json::json!({ + "source": "gateway", + "upstream": p.upstream, + "recordId": p.record_id, + "gatewayPeer": p.gateway_peer, + "acquiredAt": p.acquired_at, + })) + .expect("static JSON shape — serialisation is infallible") +} + +/// Build the flat meta-core PUT body from a [`DiscoveryRecord`] + provenance. +/// +/// Identical transform to the gateway dispatcher: any `cid_*` field becomes a +/// `cids/ = "true"` key-set member (the CID is self-describing — the +/// algorithm is the multicodec; see METADATA_KEYS.md §2/§14.13), everything else +/// passes through, and `provenance` is stamped last (overriding any plugin-set +/// value). +pub fn build_metadata_body( + record: &DiscoveryRecord, + provenance: &Provenance, +) -> BTreeMap { + let mut body: BTreeMap = BTreeMap::new(); + for (k, v) in &record.fields { + if k.starts_with("cid_") && !v.is_empty() { + body.insert(format!("cids/{v}"), "true".to_string()); + } else { + body.insert(k.clone(), v.clone()); + } + } + body.insert("provenance".to_string(), provenance_to_json(provenance)); + body +} + +/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into +/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`). +pub async fn put_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, + body: &BTreeMap, +) -> anyhow::Result<()> { + let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid); + let resp = http.put(&url).json(body).send().await?; + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("meta-core PUT {url} returned {status}: {text}"); + } + Ok(()) +} + +/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`). +#[derive(Debug, Deserialize)] +struct MetaGetResponse { + #[serde(default)] + metadata: Option>, +} + +/// `GET /meta/{cid}` — read the current flat field map for a record, or `None` +/// when meta-core has no record yet (404). Used to read back a plugin's merged +/// fields (e.g. filename-parser's `originalTitle`/`movieYear`) before handing +/// them to the next plugin in the pipeline. +pub async fn get_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, +) -> anyhow::Result>> { + let url = format!("{}/meta/{}", base_url.trim_end_matches('/'), cid); + let resp = http.get(&url).send().await?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !resp.status().is_success() { + anyhow::bail!("meta-core GET {url} returned {}", resp.status()); + } + let parsed: MetaGetResponse = resp.json().await?; + Ok(parsed.metadata) +} + +/// Self-publishing store for feeders that own their record lifecycle (D4): +/// the feeder writes its own **metadata-only** records to meta-core and returns +/// sparse outcomes so the gateway dispatcher skips them; the meta-share ingester +/// Kamilata-propagates them. +/// +/// **Byte outcomes are deliberately NOT self-published.** A feeder-side WebDAV +/// write to the gateway core would be seeded by nobody: the gateway core runs +/// with `ENABLE_FILE_WATCHER=false` (so meta-core emits no `/api/events/files` +/// for it, and the ingester's `ipfs_seed` — which seeds only watched files — +/// never sees it), and a sparse outcome makes the gateway skip its own +/// blockstore seed. So byte-bearing outcomes (torznab full-file / subtitles) +/// pass through unchanged and the **gateway** continues to WebDAV-PUT + bitswap- +/// seed them (it's the libp2p/bitswap host). Result: feeder owns the metadata +/// records (the Kamilata-propagated bulk), gateway owns byte delivery — a clean +/// split with no seeding regression. Tribler is metadata-only and uses +/// [`crate::enrich::Enricher`]; this is torznab's record-owning counterpart. +#[derive(Clone)] +pub struct FeederStore { + http: reqwest::Client, + meta_core_url: String, + gateway_peer: String, +} + +impl FeederStore { + pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self { + FeederStore { + http, + meta_core_url: meta_core_url.trim_end_matches('/').to_string(), + gateway_peer, + } + } + + /// Build from the feeder env. `None` when `META_CORE_URL` is unset (the + /// feeder then falls back to the legacy "return outcomes, gateway stores" + /// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`. + pub fn from_env() -> Option { + let meta_core_url = env_nonempty("META_CORE_URL")?; + let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()); + let http = crate::common::build_http_client(30, "meta-feeder-store", None); + Some(Self::new(http, meta_core_url, gateway_peer)) + } + + pub fn meta_core_url(&self) -> &str { + &self.meta_core_url + } + + /// Self-publish a **metadata-only** outcome to meta-core, returning the + /// **sparse** outcome (`record: None`) so the gateway's `(_, None)` branch + /// skips it — no double write. Byte outcomes and already-sparse outcomes are + /// returned unchanged (the gateway stores + seeds the bytes; see the struct + /// docs). On a write failure the original outcome is returned so the gateway + /// stores it (graceful degradation, never lost). + pub async fn publish( + &self, + upstream_id: &str, + record_id: &str, + outcome: HashOutcome, + ) -> HashOutcome { + // Only self-publish metadata-only outcomes (bytes None, record Some). + if outcome.bytes.is_some() || outcome.record.is_none() { + return outcome; + } + let cid = outcome.hash.0.clone(); + let record = outcome.record.as_ref().expect("record is Some by the guard"); + match self + .store_metadata_only(upstream_id, record_id, &cid, record) + .await + { + Ok(()) => sparse(outcome.hash, outcome.hash_kind), + Err(e) => { + warn!(target: "meta-feeder::store", cid = %cid, error = %e, + "metadata self-publish failed; falling back to gateway store"); + outcome + } + } + } + + /// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance. + async fn store_metadata_only( + &self, + upstream_id: &str, + record_id: &str, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer); + let body = build_metadata_body(record, &prov); + put_record(&self.http, &self.meta_core_url, cid, &body).await + } +} + +fn sparse(hash: Hash, hash_kind: HashKind) -> HashOutcome { + HashOutcome { + hash, + hash_kind, + bytes: None, + record: None, + file_extension: None, + } +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rec() -> DiscoveryRecord { + DiscoveryRecord { + upstream_id: "tribler".into(), + record_id: "abc".into(), + fields: BTreeMap::from_iter([ + ("title".into(), "Big Buck Bunny".into()), + ("cid_btih_v1_file".into(), "bafkPERFILE".into()), + ]), + } + } + + #[test] + fn provenance_shape_matches_gateway() { + let p = Provenance { + upstream: "tribler".into(), + record_id: "abc".into(), + gateway_peer: "metagateway-tribler".into(), + acquired_at: 1_747_504_000, + }; + let v: serde_json::Value = serde_json::from_str(&provenance_to_json(&p)).unwrap(); + assert_eq!(v["source"], "gateway"); + assert_eq!(v["upstream"], "tribler"); + assert_eq!(v["recordId"], "abc"); + assert_eq!(v["gatewayPeer"], "metagateway-tribler"); + assert_eq!(v["acquiredAt"], 1_747_504_000_u64); + } + + #[test] + fn body_collapses_cid_fields_and_stamps_provenance() { + let p = Provenance::now("tribler", "abc", "peer"); + let body = build_metadata_body(&rec(), &p); + assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny")); + // cid_* collapsed to cids/=true; no cid_* survives. + assert!(body.keys().all(|k| !k.starts_with("cid_"))); + assert_eq!( + body.get("cids/bafkPERFILE").map(String::as_str), + Some("true") + ); + assert!(body.contains_key("provenance")); + } +} diff --git a/crates/meta-feeder-sdk/src/plugin.rs b/crates/meta-feeder-sdk/src/plugin.rs new file mode 100644 index 0000000..731f35a --- /dev/null +++ b/crates/meta-feeder-sdk/src/plugin.rs @@ -0,0 +1,203 @@ +//! The `FeederPlugin` trait + supporting types. +//! +//! A feeder plugin bridges one external upstream (`gutenberg`, `arxiv`, +//! `torznab`, …) into the MetaMesh network. It **finds** records and +//! **fetches** bytes; it does NOT hash-into-a-blockstore, store back into +//! meta-core, or speak libp2p — those stay in the gateway core. This is why +//! `FeederPlugin` has no `set_blockstore` / `set_tmdb_budget` hooks (the +//! gateway's `GatewayPlugin` carried them): preview-seeding moves to the core, +//! and a feeder that needs a TMDB budget owns it internally. +//! +//! The `serve_feeders` harness ([`crate::serve`]) wraps any set of +//! `FeederPlugin`s in the feeder HTTP contract; the gateway core's +//! `RemoteFeederPlugin` is the matching HTTP client. + +use std::collections::HashMap; +use std::path::Path; + +use async_trait::async_trait; +use futures::stream::BoxStream; + +// Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the +// gateway's original `plugin.rs` re-exported it the same way). +pub use crate::query::GatewayQuery; +use crate::query::GatewaySearchEvent; +use crate::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; + +/// One element of what [`FeederPlugin::compute_outcomes`] returns. +/// +/// The `hash` is the value shipped to the gateway core. The other fields are +/// signals the core's auto-store uses to persist the content into meta-core. +/// The core's three-branch auto-store routes on `(bytes, record)`: +/// - `(Some, Some)` — full store: WebDAV PUT + metadata PUT with `filePath`. +/// - `(None, Some)` — metadata-only PUT (no local bytes; e.g. torznab samples +/// the middle 1 MiB of a multi-GiB video and surfaces only the record). +/// - `(_, None)` — skip. Cache hits land here. +/// +/// `file_extension` MUST be `None` when `bytes` is `None`. +#[derive(Debug)] +pub struct HashOutcome { + pub hash: Hash, + /// Which hash family this CID belongs to. Drives the core's routing — + /// the bitswap blockstore seed only fires for `Sha2_256`. + pub hash_kind: HashKind, + pub bytes: Option, + pub record: Option, + /// File extension to append when naming the WebDAV-side blob (e.g. + /// `"epub"`). No leading dot. MUST be `None` when `bytes` is `None`. + pub file_extension: Option, +} + +/// Hash family discriminator on [`HashOutcome`]. +/// +/// - `Midhash256` — size-prefix-plus-middle-1MB-sample, custom multicodec +/// `0x1000`. Fast, MetaMesh-internal, not retrievable via public IPFS. +/// - `Sha2_256` — standard IPFS CIDv1 over full bytes. Retrievable via bitswap +/// once the bytes are in the core's blockstore. What every shipping plugin +/// today emits. +/// - `BtV1File` — a single file inside a BitTorrent v1 torrent, custom +/// multicodec `0x1001`. Locator, not content hash: opaque to bitswap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashKind { + /// Reserved: no feeder currently emits midhash256 outcomes, but the family + /// is real across meta-core / meta-share, so the variant stays. + #[allow(dead_code)] + Midhash256, + Sha2_256, + BtV1File, +} + +/// Startup-only configuration error returned by [`FeederPlugin::configure`]. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("plugin {plugin}: required config missing: {what}")] + MissingConfig { + plugin: &'static str, + /// Human-readable description of the missing setting and where to + /// supply it. Used by the harness's soft-skip warning. + what: &'static str, + }, + + #[error("plugin {plugin}: cache dir setup failed: {source}")] + CacheSetup { + plugin: &'static str, + #[source] + source: std::io::Error, + }, + + #[error("plugin {plugin}: {source}")] + Other { + plugin: &'static str, + #[source] + source: anyhow::Error, + }, +} + +/// Canonical field name for an upstream's stable per-record identifier. +/// Every plugin's `DiscoveryRecord.fields` MUST include this key +/// (`gutenbergid:11`, `tmdbid:31910`, `arxivid:2106.07447`). The rule is +/// `format!("{upstream_id}id")` — lowercase, no separator. +pub fn upstream_id_field(upstream_id: &str) -> String { + format!("{upstream_id}id") +} + +/// Static plugin contract. Each enabled `upstream_id` is implemented by +/// exactly one `Box`. Lives behind `&self` in steady state — +/// called concurrently, owns any interior mutability for caches. +/// +/// **Field-naming convention.** Every record a plugin returns MUST include the +/// canonical `id` field — see [`upstream_id_field`]. +#[async_trait] +pub trait FeederPlugin: Send + Sync + 'static { + fn upstream_id(&self) -> &'static str; + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError>; + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError>; + + /// Streaming variant of [`handle_query`]. The default impl collects + /// [`handle_query`] and replays it as `Base*` + `Done` — correct for every + /// plugin with no incremental enrichment (gutenberg, arxiv, …). torznab + /// overrides it to decouple base records from rate-limited TMDB enrichment. + async fn handle_query_stream( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + let records = self.handle_query(query, max_results).await?; + let events = records + .into_iter() + .map(GatewaySearchEvent::Base) + .chain(std::iter::once(GatewaySearchEvent::Done)); + Ok(Box::pin(futures::stream::iter(events))) + } + + /// Resolve `record_id` into one or more content-addressed outcomes. + /// + /// - `Err(_)` — the upstream record itself is unresolvable. + /// - `Ok(vec![])` — resolved but yielded zero outcomes. + /// - `Ok(partial_vec)` — bundle-success: per-sibling failures are + /// best-effort (log + drop), surface whatever succeeded. + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError>; + + async fn handle_fetch(&self, _record_id: &str) -> Result, GatewayError> { + Ok(None) + } + + fn health(&self) -> PluginHealth { + PluginHealth::Ok + } + + fn served_file_types(&self) -> &'static [&'static str] { + &[] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &[] + } + + /// Self-describing config schema. The feeder SDK serves this at + /// `GET /config/schema` and renders a generic form from it — the gateway and + /// UI carry no per-plugin field knowledge. Default: no configuration. + fn config_schema(&self) -> crate::config::ConfigSchema { + crate::config::ConfigSchema::default() + } + + /// The plugin's current *effective* config as JSON, **unredacted** (secrets + /// included). The SDK redacts it before serving `GET /config/values` and uses + /// it as the merge base on the first save (before any `config.json` exists). + /// Default: empty object. Implementors should mirror the keys in + /// [`config_schema`](FeederPlugin::config_schema). + fn config_values(&self) -> serde_json::Value { + serde_json::json!({}) + } + + async fn get_blob(&self, _cid: &str) -> Option> { + None + } +} + +/// Static plugin registry. Built once at startup; read-only thereafter. +/// Keyed by `upstream_id`. +pub type PluginRegistry = HashMap<&'static str, Box>; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn config_error_messages_include_plugin_name() { + let e = ConfigError::MissingConfig { + plugin: "scihub", + what: + "mirrors (set them in the dashboard or gateway-config.json plugins.scihub.mirrors)", + }; + let msg = e.to_string(); + assert!(msg.contains("scihub")); + assert!(msg.contains("mirrors")); + } +} diff --git a/crates/meta-feeder-sdk/src/query.rs b/crates/meta-feeder-sdk/src/query.rs new file mode 100644 index 0000000..00f54b8 --- /dev/null +++ b/crates/meta-feeder-sdk/src/query.rs @@ -0,0 +1,147 @@ +//! Structured query envelope + streaming-search event + wire-error types, +//! shared by feeders (their HTTP contract) and the gateway core (its libp2p +//! wire). These are pure serde data types — no libp2p, no transport. +//! +//! On the core side, `protocol.rs` re-uses these exact structs as the bincode +//! wire shape; the `wire_parity_*` tests there pin the byte encoding. Keep +//! field order / variant order / derive shape stable. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::types::{DiscoveryRecord, GatewayError}; + +/// Structured query envelope. Gateway plugins do best-effort pushdown to +/// upstream APIs with native query languages (`torznab cat=movie`, +/// `arxiv id_list=`). +/// +/// **Authority rule:** `filters` / `ranges` / `negations` are authoritative; +/// `raw_text` is informational. A plugin that consumes both will double-filter +/// and under-match — pick one source per filter; structured wins. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct GatewayQuery { + /// Original user text (e.g. `"naruto contentKind:movie imdbid:tt0903747"`). + /// Verbatim — never mutated by the conversion. + pub raw_text: String, + + /// Bare-word leaves only, space-joined. The plugin's primary `q=` input + /// for upstream search APIs that take a single search box. + pub free_text: String, + + /// Structured filters keyed by field name (raw, unstemmed values). + /// Multiple values per key = OR within a key; AND across keys. + pub filters: BTreeMap>, + + /// Numeric range filters (e.g. `movieYear:2020..2024`). Both bounds + /// optional → unbounded on that side. + pub ranges: Vec, + + /// Negation filters (e.g. `NOT genres:horror`, bare `NOT word`). + pub negations: Vec, +} + +/// Numeric range filter inside [`GatewayQuery::ranges`]. Both bounds are +/// `Option` so open-ended ranges drop one bound to `None`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RangeFilter { + pub field: String, + pub lo: Option, + pub hi: Option, +} + +/// Negation entry inside [`GatewayQuery::negations`]. `field` is `Some(name)` +/// for typed negations (`NOT genres:horror`), `None` for bare-word negation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Negation { + pub field: Option, + pub value: String, +} + +impl GatewayQuery { + /// Bare-text input to hand to a free-form upstream search API. Falls back + /// to `"*"` when the conversion produced no free-text leaves so upstreams + /// that 4xx on empty search return their top-ranked results instead. + pub fn free_text_or_star(&self) -> &str { + if self.free_text.trim().is_empty() { + "*" + } else { + self.free_text.as_str() + } + } + + /// Construct a free-text-only `GatewayQuery` for in-process use (tests, + /// debug routes). `raw_text` == `free_text`; structured fields stay empty. + pub fn from_free_text(text: impl Into) -> Self { + let text = text.into(); + Self { + raw_text: text.clone(), + free_text: text, + filters: BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + } + } +} + +/// One frame in the streaming Search response. +/// +/// A plugin emits `Base` records as soon as they're discovered, then +/// best-effort `EnrichPatch`/`Drop` events as enrichment lands, then a +/// terminal `Done`. The default `handle_query_stream` collects `handle_query` +/// and replays it as `Base*` + `Done`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum GatewaySearchEvent { + /// An un-enriched discovery record. + Base(DiscoveryRecord), + /// Incremental enrichment for a previously-sent `Base`, keyed by + /// `record_id`. `set` fields are inserted/overwritten; `remove` deleted. + EnrichPatch { + record_id: String, + set: BTreeMap, + remove: Vec, + }, + /// Retract a previously-emitted `Base` record entirely. + Drop { record_id: String }, + /// Terminal success — no more frames follow. + Done, + /// Terminal error — no more frames follow. + Error(GatewayWireError), +} + +/// Wire-stable error shape. Distinct from in-process [`GatewayError`] because +/// adding a variant here is a wire-format change. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, thiserror::Error)] +pub enum GatewayWireError { + #[error("not found")] + NotFound, + #[error("rate limited; retry in {retry_after_s}s")] + RateLimited { retry_after_s: u32 }, + #[error("upstream transient: {0}")] + UpstreamTransient(String), + #[error("upstream permanent: {0}")] + UpstreamPermanent(String), + /// The protocol was registered but no plugin claims this `upstream_id`. + #[error("plugin not loaded")] + PluginNotLoaded, + /// Internal codec / decode failure on the *receiver* side. + #[error("internal: {0}")] + Internal(String), +} + +impl From for GatewayWireError { + fn from(e: GatewayError) -> Self { + match e { + GatewayError::Transient(s) => GatewayWireError::UpstreamTransient(s), + GatewayError::Permanent(s) => GatewayWireError::UpstreamPermanent(s), + GatewayError::RateLimited { retry_after_s } => { + GatewayWireError::RateLimited { retry_after_s } + } + GatewayError::NotFound => GatewayWireError::NotFound, + // Internal errors are never leaked; map to a transient. + GatewayError::Internal(_) => { + GatewayWireError::UpstreamTransient("internal".to_string()) + } + } + } +} diff --git a/crates/meta-feeder-sdk/src/query_eval.rs b/crates/meta-feeder-sdk/src/query_eval.rs new file mode 100644 index 0000000..523187d --- /dev/null +++ b/crates/meta-feeder-sdk/src/query_eval.rs @@ -0,0 +1,456 @@ +//! Structured-filter evaluation for gateway records. +//! +//! Two pure helpers consuming the [`crate::query::GatewayQuery`] struct: +//! +//! - [`record_matches`] — does a single record's `fields` map satisfy the +//! query's `filters` + `ranges` + typed `negations`? +//! - [`query_accepts_plugin`] — could a plugin advertising the given +//! `served_file_types` / `served_content_kinds` possibly produce a record +//! that satisfies the query? Used by plugins as a cheap early-return. +//! +//! The dispatcher applies `record_matches` to every plugin's return value, and +//! meta-share applies the same algorithm consumer-side to defend against stale +//! gateway peers — both tiers must agree on outcomes for any (record, query) +//! pair. See the original gateway crate docs for the full semantics. + +use std::collections::BTreeMap; + +use crate::query::GatewayQuery; + +/// True iff a record's `fields` satisfies every filter, range, and typed +/// negation in `query`. +pub fn record_matches(fields: &BTreeMap, query: &GatewayQuery) -> bool { + for (key, allowed) in &query.filters { + if key == "languages" { + if !languages_filter_matches(fields, allowed) { + return false; + } + continue; + } + let Some(field_value) = fields.get(key) else { + return false; + }; + if !any_csv_value_matches(field_value, allowed) { + return false; + } + } + + for range in &query.ranges { + let Some(field_value) = fields.get(&range.field) else { + return false; + }; + let Ok(n) = field_value.parse::() else { + return false; + }; + if let Some(lo) = range.lo { + if n < lo { + return false; + } + } + if let Some(hi) = range.hi { + if n > hi { + return false; + } + } + } + + for negation in &query.negations { + let Some(name) = negation.field.as_deref() else { + // Bare-word negation — not enforced here. + continue; + }; + let Some(field_value) = fields.get(name) else { + // Missing field ⇒ negation passes (nothing to exclude). + continue; + }; + if any_csv_value_matches(field_value, std::slice::from_ref(&negation.value)) { + return false; + } + } + + true +} + +/// True iff a plugin advertising the given `served_file_types` / +/// `served_content_kinds` could possibly produce a record satisfying +/// `query.filters["fileType" | "contentKind"]`. +pub fn query_accepts_plugin( + query: &GatewayQuery, + served_file_types: &[&str], + served_content_kinds: &[&str], +) -> bool { + accepts_axis(query.filters.get("fileType"), served_file_types) + && accepts_axis(query.filters.get("contentKind"), served_content_kinds) +} + +fn accepts_axis(requested: Option<&Vec>, served: &[&str]) -> bool { + let Some(requested) = requested else { + return true; + }; + if requested.is_empty() { + return true; + } + // Wildcard: a plugin that advertises `"*"` on an axis serves every value on + // that axis (e.g. tribler is a free-text overlay search with no inherent + // type limitation). It accepts any requested fileType/contentKind. + if served.contains(&"*") { + return true; + } + requested + .iter() + .any(|v| served.iter().any(|s| s.eq_ignore_ascii_case(v))) +} + +/// Resolve a `languages` filter against the key-set storage form +/// (`languages/ = "true"`). **Fails open** on anything we can't +/// confidently exclude. A record matches when: +/// 1. it carries a requested code as a truthy key-set member (OR across codes); +/// 2. a legacy flat `languages` csv field contains a requested code; +/// 3. it carries `mul` (multiple — may contain a requested one) or `und` +/// (undetermined); or +/// 4. it carries no language information at all. +/// +/// Only a record with a *concrete*, non-matching language (e.g. `jpn`-only +/// under an `eng` filter) is dropped. Kept byte-identical in logic with +/// meta-search's copy (and the client's `passesLangFilter`) — language narrows +/// the result set without ever hiding unknown/undetermined/multi releases. +fn languages_filter_matches(fields: &BTreeMap, allowed: &[String]) -> bool { + for code in allowed { + let key = format!("languages/{}", code.trim().to_ascii_lowercase()); + if fields.get(&key).is_some_and(|v| is_truthy_member(v)) { + return true; + } + } + if let Some(flat) = fields.get("languages") { + if any_csv_value_matches(flat, allowed) { + return true; + } + } + // Fail open: `mul`/`und` are explicit "could be anything / unknown" markers, + // and a record with no concrete language at all is equally unknown. Keep all + // three; only a specific, non-matching language reaches the `false` below. + if fields.get("languages/mul").is_some_and(|v| is_truthy_member(v)) + || fields.get("languages/und").is_some_and(|v| is_truthy_member(v)) + { + return true; + } + !record_has_concrete_language(fields) +} + +/// True iff the record states a *concrete* language: a truthy `languages/` +/// key-set member other than the `und`/`mul` non-codes, or a non-empty legacy +/// flat `languages` field. Distinguishes "a specific language that didn't match +/// ⇒ drop" from "no language info ⇒ keep" for [`languages_filter_matches`]. +fn record_has_concrete_language(fields: &BTreeMap) -> bool { + if fields.get("languages").is_some_and(|v| !v.trim().is_empty()) { + return true; + } + fields.iter().any(|(k, v)| { + is_truthy_member(v) + && k.starts_with("languages/") + && k != "languages/und" + && k != "languages/mul" + }) +} + +/// Key-set members are written as the literal `"true"`; tolerate a couple of +/// obvious truthy spellings and treat anything else as absent. +fn is_truthy_member(v: &str) -> bool { + matches!(v.trim().to_ascii_lowercase().as_str(), "true" | "1" | "yes") +} + +/// True iff `field_value` (treated as a csv set when it contains commas) +/// contains at least one case-insensitive match for any item in `allowed`. +fn any_csv_value_matches(field_value: &str, allowed: &[String]) -> bool { + for piece in field_value.split(',') { + let piece = piece.trim(); + if piece.is_empty() { + continue; + } + if allowed.iter().any(|v| v.eq_ignore_ascii_case(piece)) { + return true; + } + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::query::{Negation, RangeFilter}; + + fn empty_query() -> GatewayQuery { + GatewayQuery { + raw_text: String::new(), + free_text: String::new(), + filters: BTreeMap::new(), + ranges: Vec::new(), + negations: Vec::new(), + } + } + + fn with_filter(field: &str, values: &[&str]) -> GatewayQuery { + let mut q = empty_query(); + q.filters.insert( + field.to_string(), + values.iter().map(|s| s.to_string()).collect(), + ); + q + } + + fn fields(pairs: &[(&str, &str)]) -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + #[test] + fn filter_rejects_mismatched_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + } + + #[test] + fn filter_accepts_matching_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + } + + #[test] + fn filter_rejects_record_missing_field() { + let q = with_filter("fileType", &["video"]); + assert!(!record_matches(&fields(&[]), &q)); + } + + #[test] + fn filter_is_case_insensitive() { + let q = with_filter("fileType", &["VIDEO"]); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + let q = with_filter("fileType", &["video"]); + assert!(record_matches(&fields(&[("fileType", "VIDEO")]), &q)); + } + + #[test] + fn filter_ors_within_key() { + let q = with_filter("fileType", &["video", "audio"]); + assert!(record_matches(&fields(&[("fileType", "audio")]), &q)); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + } + + #[test] + fn filter_ands_across_keys() { + let mut q = with_filter("fileType", &["video"]); + q.filters.insert("genres".into(), vec!["Action".into()]); + assert!(!record_matches( + &fields(&[("fileType", "video"), ("genres", "Drama")]), + &q + )); + assert!(record_matches( + &fields(&[("fileType", "video"), ("genres", "Action")]), + &q + )); + } + + #[test] + fn csv_field_matches_one_of_its_values() { + let q = with_filter("genres", &["Anime"]); + assert!(record_matches( + &fields(&[("genres", "Animation,Anime,Drama")]), + &q + )); + } + + #[test] + fn csv_field_with_whitespace_is_trimmed() { + let q = with_filter("genres", &["Anime"]); + assert!(record_matches( + &fields(&[("genres", "Animation, Anime , Drama")]), + &q + )); + } + + #[test] + fn languages_keyset_member_matches() { + let q = with_filter("languages", &["eng"]); + assert!(record_matches(&fields(&[("languages/eng", "true")]), &q)); + } + + #[test] + fn languages_filter_is_or_across_codes() { + let q = with_filter("languages", &["eng", "fre"]); + assert!(record_matches(&fields(&[("languages/eng", "true")]), &q)); + assert!(record_matches(&fields(&[("languages/fre", "true")]), &q)); + assert!(!record_matches(&fields(&[("languages/jpn", "true")]), &q)); + } + + #[test] + fn languages_filter_fails_open_on_unknown() { + let q = with_filter("languages", &["eng"]); + // No language info at all ⇒ kept (unknown, not excluded). + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + // `mul` (multi) may contain the requested language ⇒ kept. + assert!(record_matches(&fields(&[("languages/mul", "true")]), &q)); + // `und` (undetermined) ⇒ kept. + assert!(record_matches(&fields(&[("languages/und", "true")]), &q)); + // A specific language alongside `mul` is still kept (multi). + assert!(record_matches( + &fields(&[("languages/jpn", "true"), ("languages/mul", "true")]), + &q + )); + } + + #[test] + fn languages_concrete_mismatch_still_drops() { + let q = with_filter("languages", &["eng"]); + // A concrete, non-matching language ⇒ dropped (language narrows). + assert!(!record_matches(&fields(&[("languages/jpn", "true")]), &q)); + // A tombstoned/false member is treated as absent ⇒ unknown ⇒ kept. + assert!(record_matches(&fields(&[("languages/eng", "false")]), &q)); + } + + #[test] + fn languages_legacy_flat_csv_fallback() { + let q = with_filter("languages", &["eng"]); + assert!(record_matches(&fields(&[("languages", "jpn,eng")]), &q)); + assert!(!record_matches(&fields(&[("languages", "jpn,ger")]), &q)); + } + + #[test] + fn range_in_bounds() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: Some(2020), + }); + assert!(record_matches(&fields(&[("movieYear", "2015")]), &q)); + } + + #[test] + fn range_out_of_bounds_rejected() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: Some(2020), + }); + assert!(!record_matches(&fields(&[("movieYear", "2025")]), &q)); + assert!(!record_matches(&fields(&[("movieYear", "2005")]), &q)); + } + + #[test] + fn range_open_upper_bound() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: None, + }); + assert!(record_matches(&fields(&[("movieYear", "2025")]), &q)); + } + + #[test] + fn range_unparseable_value_rejected() { + let mut q = empty_query(); + q.ranges.push(RangeFilter { + field: "movieYear".into(), + lo: Some(2010), + hi: None, + }); + assert!(!record_matches(&fields(&[("movieYear", "circa 2015")]), &q)); + } + + #[test] + fn typed_negation_excludes_match() { + let mut q = empty_query(); + q.negations.push(Negation { + field: Some("fileType".into()), + value: "image".into(), + }); + assert!(!record_matches(&fields(&[("fileType", "image")]), &q)); + assert!(record_matches(&fields(&[("fileType", "video")]), &q)); + } + + #[test] + fn typed_negation_passes_when_field_missing() { + let mut q = empty_query(); + q.negations.push(Negation { + field: Some("fileType".into()), + value: "image".into(), + }); + assert!(record_matches(&fields(&[]), &q)); + } + + #[test] + fn bare_word_negation_is_not_enforced() { + let mut q = empty_query(); + q.negations.push(Negation { + field: None, + value: "alice".into(), + }); + assert!(record_matches( + &fields(&[("title", "alice in wonderland")]), + &q + )); + } + + #[test] + fn empty_query_matches_anything() { + assert!(record_matches(&fields(&[]), &empty_query())); + assert!(record_matches( + &fields(&[("fileType", "video"), ("title", "x")]), + &empty_query() + )); + } + + #[test] + fn accepts_returns_true_when_no_type_filter() { + let q = empty_query(); + assert!(query_accepts_plugin(&q, &["video"], &["movie"])); + assert!(query_accepts_plugin(&q, &[], &[])); + } + + #[test] + fn accepts_rejects_disjoint_filetype() { + let q = with_filter("fileType", &["video"]); + assert!(!query_accepts_plugin(&q, &["image"], &["gif"])); + assert!(query_accepts_plugin(&q, &["video"], &[])); + } + + #[test] + fn accepts_rejects_disjoint_contentkind() { + let q = with_filter("contentKind", &["movie"]); + assert!(!query_accepts_plugin( + &q, + &["video", "image"], + &["paper", "book"] + )); + assert!(query_accepts_plugin(&q, &["video"], &["movie", "episode"])); + } + + #[test] + fn accepts_requires_both_axes_when_both_filtered() { + let mut q = with_filter("fileType", &["video"]); + q.filters.insert("contentKind".into(), vec!["movie".into()]); + assert!(query_accepts_plugin(&q, &["video"], &["movie"])); + assert!(!query_accepts_plugin(&q, &["video"], &["paper"])); + assert!(!query_accepts_plugin(&q, &["image"], &["movie"])); + } + + #[test] + fn accepts_wildcard_matches_any_requested_type() { + // A plugin advertising "*" (e.g. tribler) accepts any fileType / + // contentKind filter on the wildcarded axis. + let q = with_filter("fileType", &["archive"]); + assert!(query_accepts_plugin(&q, &["*"], &[])); + let q = with_filter("contentKind", &["audiobook"]); + assert!(query_accepts_plugin(&q, &["*"], &["*"])); + // Wildcard on one axis only: the non-wildcard axis still gates. + let mut q = with_filter("fileType", &["audio"]); + q.filters.insert("contentKind".into(), vec!["podcast".into()]); + assert!(query_accepts_plugin(&q, &["*"], &["*"])); + assert!(!query_accepts_plugin(&q, &["*"], &["movie"])); + } +} diff --git a/crates/meta-feeder-sdk/src/serve.rs b/crates/meta-feeder-sdk/src/serve.rs new file mode 100644 index 0000000..f0b0421 --- /dev/null +++ b/crates/meta-feeder-sdk/src/serve.rs @@ -0,0 +1,634 @@ +//! `serve_feeders` — the axum harness that exposes a set of [`FeederPlugin`]s +//! over the **feeder HTTP contract**. Every feeder sidecar binary is just +//! `serve_feeders(vec![MyPlugin::new()], state_dir, listen)`. +//! +//! # Contract (JSON over HTTP, mirrors meta-sort's container-plugin shape) +//! +//! - `GET /manifest` → [`ManifestResponse`] +//! - `GET /health` → [`HealthResponse`] +//! - `POST /query` → [`QueryRequest`] → [`QueryResponse`] +//! - `POST /compute` → [`ComputeRequest`] → [`ComputeResponse`] +//! - `GET /fetch/{upstream_id}/{record_id}` → streamed bytes (or 404) +//! - `GET /blob/{upstream_id}/{cid}` → bytes (or 404) +//! +//! ## Bytes & the v1 simplification +//! +//! `/compute` returns each outcome's bytes **inline, base64-encoded** +//! ([`OutcomeDto::bytes_b64`]) — this reuses every existing `compute_outcomes` +//! impl verbatim (they already return bytes in `HashOutcome.bytes`) and keeps +//! the gateway core's auto-store a single round-trip. base64 adds ~33%; fine +//! for books/papers/posters. Large-file streaming (torznab multi-GiB) is a +//! documented follow-up — those outcomes are metadata-only anyway +//! (`bytes: None`), so they never ride this path. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::path::{Path as FsPath, PathBuf}; +use std::sync::Arc; + +use axum::{ + body::Body, + extract::{Path, State}, + http::StatusCode, + response::{Html, IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use base64::Engine as _; +use futures::StreamExt; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{info, warn}; + +use crate::config::{self, CONFIG_PAGE_HTML}; +use crate::plugin::{ConfigError, FeederPlugin, HashKind, HashOutcome}; +use crate::query::{GatewayQuery, GatewaySearchEvent, GatewayWireError}; +use crate::types::{DiscoveryRecord, GatewayError, PluginHealth}; + +// -------- wire DTOs -------------------------------------------------------- + +/// Serializable mirror of [`HashKind`] for the feeder HTTP boundary. +#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)] +#[serde(rename_all = "snake_case")] +pub enum HashKindDto { + Midhash256, + Sha2_256, + BtV1File, +} + +impl From for HashKindDto { + fn from(k: HashKind) -> Self { + match k { + HashKind::Midhash256 => HashKindDto::Midhash256, + HashKind::Sha2_256 => HashKindDto::Sha2_256, + HashKind::BtV1File => HashKindDto::BtV1File, + } + } +} + +impl From for HashKind { + fn from(k: HashKindDto) -> Self { + match k { + HashKindDto::Midhash256 => HashKind::Midhash256, + HashKindDto::Sha2_256 => HashKind::Sha2_256, + HashKindDto::BtV1File => HashKind::BtV1File, + } + } +} + +/// One plugin's capability surface in [`ManifestResponse`]. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct PluginManifest { + pub id: String, + pub served_file_types: Vec, + pub served_content_kinds: Vec, +} + +/// `GET /manifest` — the feeder's hosted plugins + their served types. The +/// gateway core reads this to register `RemoteFeederPlugin`s and to build the +/// capability heartbeat it broadcasts to meta-share. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ManifestResponse { + pub feeder_version: String, + pub plugins: Vec, +} + +/// One plugin's liveness in [`HealthResponse`]. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct PluginHealthEntry { + pub id: String, + pub health: PluginHealth, +} + +/// `GET /health` — `status` is `"ok"` iff every hosted plugin reports `Ok`. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct HealthResponse { + pub status: String, + pub plugins: Vec, +} + +/// `POST /query` request body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct QueryRequest { + pub upstream_id: String, + pub query: GatewayQuery, + pub max_results: u32, +} + +/// `POST /query` response body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct QueryResponse { + pub records: Vec, +} + +/// `POST /compute` request body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ComputeRequest { + pub upstream_id: String, + pub record_id: String, +} + +/// One resolved outcome on the feeder wire. `bytes_b64` carries the full +/// payload base64-encoded when the plugin fetched bytes (`Sha2_256` full +/// store); `None` for metadata-only outcomes (the core's `(None, Some)` +/// branch — e.g. torznab midhash). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct OutcomeDto { + pub hash: String, + pub hash_kind: HashKindDto, + pub file_extension: Option, + pub record: Option, + pub bytes_b64: Option, +} + +impl OutcomeDto { + fn from_outcome(o: HashOutcome) -> Self { + OutcomeDto { + hash: o.hash.0, + hash_kind: o.hash_kind.into(), + file_extension: o.file_extension, + record: o.record, + bytes_b64: o + .bytes + .map(|b| base64::engine::general_purpose::STANDARD.encode(&b)), + } + } +} + +/// `POST /compute` response body. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ComputeResponse { + pub outcomes: Vec, +} + +// -------- app state + error mapping ---------------------------------------- + +#[derive(Clone)] +struct AppState { + plugins: Arc>>, + version: String, + /// Feeder state dir. Per-plugin config persists at + /// `state_dir/gateway//config.json` — the same per-plugin dir handed to + /// `configure()`, so a UI save is read back on the next restart. + state_dir: PathBuf, +} + +impl AppState { + fn plugin(&self, upstream_id: &str) -> Result<&Arc, Response> { + self.plugins.get(upstream_id).ok_or_else(|| { + ( + StatusCode::NOT_FOUND, + format!("no feeder plugin for upstream_id '{upstream_id}'"), + ) + .into_response() + }) + } + + /// `state_dir/gateway//config.json` — canonical per-plugin config file. + fn config_path(&self, upstream_id: &str) -> PathBuf { + self.state_dir + .join("gateway") + .join(upstream_id) + .join("config.json") + } + + /// The effective config as JSON: the persisted `config.json` if it exists, + /// else the plugin's in-memory (env-seeded) config. Used as the redaction + /// source for reads and the merge base for writes. + fn effective_config(&self, upstream_id: &str) -> Value { + if let Ok(bytes) = std::fs::read(self.config_path(upstream_id)) { + if let Ok(v) = serde_json::from_slice::(&bytes) { + return v; + } + } + self.plugins + .get(upstream_id) + .map(|p| p.config_values()) + .unwrap_or_else(|| serde_json::json!({})) + } +} + +/// Map an in-process [`GatewayError`] to an HTTP status + message. Mirrors the +/// status semantics the gateway's `RemoteFeederPlugin` will translate back +/// into a `GatewayError` on the core side. +fn gateway_error_response(e: GatewayError) -> Response { + let (status, msg) = match e { + GatewayError::NotFound => (StatusCode::NOT_FOUND, "not found".to_string()), + GatewayError::RateLimited { retry_after_s } => ( + StatusCode::TOO_MANY_REQUESTS, + format!("rate limited; retry in {retry_after_s}s"), + ), + GatewayError::Transient(s) => (StatusCode::BAD_GATEWAY, s), + GatewayError::Permanent(s) => (StatusCode::UNPROCESSABLE_ENTITY, s), + GatewayError::Internal(s) => (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()), + }; + (status, msg).into_response() +} + +// -------- handlers --------------------------------------------------------- + +async fn manifest(State(state): State) -> Json { + let mut plugins: Vec = state + .plugins + .values() + .map(|p| PluginManifest { + id: p.upstream_id().to_string(), + served_file_types: p.served_file_types().iter().map(|s| s.to_string()).collect(), + served_content_kinds: p + .served_content_kinds() + .iter() + .map(|s| s.to_string()) + .collect(), + }) + .collect(); + plugins.sort_by(|a, b| a.id.cmp(&b.id)); + Json(ManifestResponse { + feeder_version: state.version.clone(), + plugins, + }) +} + +async fn health(State(state): State) -> Json { + let mut entries: Vec = state + .plugins + .values() + .map(|p| PluginHealthEntry { + id: p.upstream_id().to_string(), + health: p.health(), + }) + .collect(); + entries.sort_by(|a, b| a.id.cmp(&b.id)); + let all_ok = entries + .iter() + .all(|e| matches!(e.health, PluginHealth::Ok)); + Json(HealthResponse { + status: if all_ok { "ok" } else { "degraded" }.to_string(), + plugins: entries, + }) +} + +async fn query( + State(state): State, + Json(req): Json, +) -> Result, Response> { + let plugin = state.plugin(&req.upstream_id)?; + let records = plugin + .handle_query(&req.query, req.max_results as usize) + .await + .map_err(gateway_error_response)?; + Ok(Json(QueryResponse { records })) +} + +/// `POST /query_stream` — the streaming analog of [`query`]. Emits the plugin's +/// [`FeederPlugin::handle_query_stream`] events as **NDJSON**: one JSON-encoded +/// [`GatewaySearchEvent`] per line (`application/x-ndjson`). The gateway core's +/// `RemoteFeederPlugin` consumes this and re-emits a `BoxStream` +/// into the libp2p streaming worker, preserving incremental enrichment (e.g. +/// torznab's late TMDB / poster / subtitle-availability patches). A plugin that +/// doesn't override `handle_query_stream` still works here — the trait default +/// replays `handle_query` as `Base* + Done`. +async fn query_stream( + State(state): State, + Json(req): Json, +) -> Result { + let plugin = state.plugin(&req.upstream_id)?; + // The open can fail (e.g. the plugin can't reach its upstream at all); that + // maps to an HTTP error status, same as `/query`. Per-event errors instead + // ride the stream as a terminal `GatewaySearchEvent::Error` line. + let events = plugin + .handle_query_stream(&req.query, req.max_results as usize) + .await + .map_err(gateway_error_response)?; + let body = events.map(|ev| { + // Serialization of GatewaySearchEvent is infallible in practice; on the + // off chance it fails, terminate with an Error line so the consumer sees + // a clean terminal failure rather than a truncated stream. + let mut line = serde_json::to_vec(&ev).unwrap_or_else(|e| { + serde_json::to_vec(&GatewaySearchEvent::Error(GatewayWireError::Internal( + format!("event serialize: {e}"), + ))) + .unwrap_or_default() + }); + line.push(b'\n'); + Ok::<_, std::convert::Infallible>(line) + }); + Ok(( + [(axum::http::header::CONTENT_TYPE, "application/x-ndjson")], + Body::from_stream(body), + ) + .into_response()) +} + +async fn compute( + State(state): State, + Json(req): Json, +) -> Result, Response> { + let plugin = state.plugin(&req.upstream_id)?; + let outcomes = plugin + .compute_outcomes(&req.record_id) + .await + .map_err(gateway_error_response)?; + Ok(Json(ComputeResponse { + outcomes: outcomes.into_iter().map(OutcomeDto::from_outcome).collect(), + })) +} + +async fn fetch( + State(state): State, + Path((upstream_id, record_id)): Path<(String, String)>, +) -> Result { + let plugin = state.plugin(&upstream_id)?; + match plugin + .handle_fetch(&record_id) + .await + .map_err(gateway_error_response)? + { + Some(stream) => { + // Map the plugin's GatewayError stream into an io::Error stream so + // axum's Body::from_stream accepts it. + let body_stream = stream.map(|chunk| { + chunk.map_err(|e| std::io::Error::other(e.to_string())) + }); + Ok(Body::from_stream(body_stream).into_response()) + } + None => Err((StatusCode::NOT_FOUND, "no bytes for record").into_response()), + } +} + +async fn blob( + State(state): State, + Path((upstream_id, cid)): Path<(String, String)>, +) -> Result { + let plugin = state.plugin(&upstream_id)?; + match plugin.get_blob(&cid).await { + Some(bytes) => Ok(bytes.into_response()), + None => Err((StatusCode::NOT_FOUND, "no such blob").into_response()), + } +} + +// -------- config plane ----------------------------------------------------- +// +// Generic, schema-driven per-plugin config. The page + the schema/values JSON +// are all served relative to the request path, so this works identically when +// hit directly on the feeder or reverse-proxied through the gateway. + +/// `GET /config` — the schema-driven HTML form (static, plugin-agnostic). +async fn config_page() -> Html<&'static str> { + Html(CONFIG_PAGE_HTML) +} + +/// `GET /config/schema` — the single hosted plugin's declared config schema. +/// (A feeder hosts one plugin in practice; if it hosts several, the first by +/// id is used — the gateway proxies per upstream_id anyway.) +async fn config_schema(State(state): State) -> Json { + let schema = sole_plugin(&state) + .map(|p| p.config_schema()) + .unwrap_or_default(); + Json(schema) +} + +/// `GET /config/values` — current values with secrets redacted to `_set`. +async fn config_values_get(State(state): State) -> Json { + match sole_plugin(&state) { + Some(p) => { + let schema = p.config_schema(); + let eff = state.effective_config(p.upstream_id()); + Json(config::redact(&eff, &schema)) + } + None => Json(serde_json::json!({})), + } +} + +/// `PUT /config/values` — merge the submitted values onto the stored config +/// (schema-aware; blank secrets keep their stored value), persist atomically, +/// and signal that a feeder restart is needed to apply them. +async fn config_values_put( + State(state): State, + Json(incoming): Json, +) -> Result, Response> { + let plugin = sole_plugin(&state) + .ok_or_else(|| (StatusCode::NOT_FOUND, "feeder hosts no plugin").into_response())?; + let id = plugin.upstream_id(); + let schema = plugin.config_schema(); + let base = state.effective_config(id); + let merged = config::merge(&base, &incoming, &schema); + + let path = state.config_path(id); + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("mkdir: {e}")).into_response())?; + } + let tmp = path.with_extension("json.tmp"); + let body = serde_json::to_vec_pretty(&merged) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("serialize: {e}")).into_response())?; + std::fs::write(&tmp, body) + .and_then(|_| std::fs::rename(&tmp, &path)) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("write config: {e}")).into_response())?; + info!(target: "meta-feeder", upstream_id = id, path = %path.display(), "feeder config saved (restart to apply)"); + Ok(Json(serde_json::json!({ "saved": true, "restart_required": true }))) +} + +/// `POST /enrich/callback` — sink for the enrichment plugins' completion +/// callbacks. The feeder drives those plugins (filename-parser, tmdb) during a +/// self-published resolve and reads results back by polling meta-core, so the +/// callback carries no load — this route exists only so the plugins' callback +/// POST gets a 200 instead of logging a connection failure. Body is ignored. +async fn enrich_callback() -> StatusCode { + StatusCode::OK +} + +/// A feeder hosts one plugin in the common case. Pick it (lowest id for +/// determinism if several are present). +fn sole_plugin(state: &AppState) -> Option<&Arc> { + state + .plugins + .values() + .min_by(|a, b| a.upstream_id().cmp(b.upstream_id())) +} + +// -------- entry point ------------------------------------------------------ + +/// Build the feeder router for an already-configured plugin set. Exposed for +/// in-process tests (drive it without binding a socket). +pub fn router( + plugins: HashMap>, + version: String, + state_dir: impl Into, +) -> Router { + let state = AppState { + plugins: Arc::new(plugins), + version, + state_dir: state_dir.into(), + }; + Router::new() + .route("/manifest", get(manifest)) + .route("/health", get(health)) + .route("/query", post(query)) + .route("/query_stream", post(query_stream)) + .route("/compute", post(compute)) + .route("/fetch/:upstream_id/:record_id", get(fetch)) + .route("/blob/:upstream_id/:cid", get(blob)) + .route("/config", get(config_page)) + .route("/config/schema", get(config_schema)) + .route("/config/values", get(config_values_get).put(config_values_put)) + .route("/enrich/callback", post(enrich_callback)) + .with_state(state) +} + +/// Configure each plugin (cache dir under `state_dir/gateway/`), then serve +/// the feeder HTTP contract on `listen` until shutdown. +/// +/// A plugin whose `configure()` returns [`ConfigError::MissingConfig`] is +/// **soft-skipped** (warn + dropped) — mirroring the gateway registry's +/// behaviour, so a misconfigured plugin doesn't take the whole feeder down. +/// Any other `ConfigError` aborts startup. +pub async fn serve_feeders( + plugins: Vec>, + state_dir: impl Into, + listen: SocketAddr, +) -> anyhow::Result<()> { + let state_dir = state_dir.into(); + let configured = configure_plugins(plugins, &state_dir)?; + let version = env!("CARGO_PKG_VERSION").to_string(); + let app = router(configured, version, state_dir.clone()); + + let listener = tokio::net::TcpListener::bind(listen).await?; + info!( + target: "meta-feeder", + %listen, + "feeder listening" + ); + axum::serve(listener, app) + .await + .map_err(anyhow::Error::from) +} + +/// Run each plugin's `configure()` against its per-plugin cache dir, applying +/// the soft-skip rule, and return the surviving plugins keyed by `upstream_id`. +/// Public so feeder binaries / tests can configure without serving. +pub fn configure_plugins( + plugins: Vec>, + state_dir: &FsPath, +) -> anyhow::Result>> { + let mut out: HashMap> = HashMap::new(); + for mut plugin in plugins { + let id = plugin.upstream_id(); + let cache_dir = state_dir.join("gateway").join(id); + std::fs::create_dir_all(&cache_dir).map_err(|e| { + anyhow::anyhow!("create cache dir {} for {id}: {e}", cache_dir.display()) + })?; + match plugin.configure(&cache_dir) { + Ok(()) => { + let id = id.to_string(); + out.insert(id, Arc::from(plugin)); + } + Err(ConfigError::MissingConfig { plugin: p, what }) => { + warn!( + target: "meta-feeder", + upstream_id = p, + missing = what, + "feeder plugin skipped: required config not supplied; \ + other plugins continue" + ); + } + Err(other) => return Err(anyhow::anyhow!(other)), + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::DiscoveryRecord; + use async_trait::async_trait; + use futures::stream::BoxStream; + use std::collections::BTreeMap; + + /// A feeder plugin that overrides `handle_query_stream` to emit a Base, then + /// an EnrichPatch, then Done — so the NDJSON route is exercised on a real + /// incremental stream (not just the collect-default). + struct StubStreaming; + + #[async_trait] + impl FeederPlugin for StubStreaming { + fn upstream_id(&self) -> &'static str { + "stub" + } + fn configure(&mut self, _cache_dir: &FsPath) -> Result<(), ConfigError> { + Ok(()) + } + async fn handle_query( + &self, + _q: &GatewayQuery, + _n: usize, + ) -> Result, GatewayError> { + Ok(vec![]) + } + async fn handle_query_stream( + &self, + _q: &GatewayQuery, + _n: usize, + ) -> Result, GatewayError> { + let rec = DiscoveryRecord { + upstream_id: "stub".into(), + record_id: "1".into(), + fields: BTreeMap::new(), + }; + let events = vec![ + GatewaySearchEvent::Base(rec), + GatewaySearchEvent::EnrichPatch { + record_id: "1".into(), + set: BTreeMap::from([("title".to_string(), "X".to_string())]), + remove: vec![], + }, + GatewaySearchEvent::Done, + ]; + Ok(Box::pin(futures::stream::iter(events))) + } + async fn compute_outcomes( + &self, + _record_id: &str, + ) -> Result, GatewayError> { + Ok(vec![]) + } + } + + #[tokio::test] + async fn query_stream_emits_ndjson_events_in_order() { + let mut plugins: HashMap> = HashMap::new(); + plugins.insert("stub".to_string(), Arc::new(StubStreaming)); + let app = router(plugins, "test".to_string(), std::env::temp_dir()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let body = reqwest::Client::new() + .post(format!("http://{addr}/query_stream")) + .json(&serde_json::json!({ + "upstream_id": "stub", + "query": {"raw_text":"x","free_text":"x","filters":{},"ranges":[],"negations":[]}, + "max_results": 5 + })) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + let events: Vec = body + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| serde_json::from_str(l).expect("each NDJSON line is a GatewaySearchEvent")) + .collect(); + + assert_eq!(events.len(), 3, "body was: {body:?}"); + assert!(matches!(events[0], GatewaySearchEvent::Base(_))); + assert!(matches!(events[1], GatewaySearchEvent::EnrichPatch { .. })); + assert!(matches!(events[2], GatewaySearchEvent::Done)); + } +} diff --git a/crates/meta-feeder-sdk/src/types.rs b/crates/meta-feeder-sdk/src/types.rs new file mode 100644 index 0000000..9fbc7b6 --- /dev/null +++ b/crates/meta-feeder-sdk/src/types.rs @@ -0,0 +1,124 @@ +//! In-process types used by the gateway runtime, feeders, and plugins. +//! +//! `GatewayError` here is **distinct from** the wire-stable +//! `GatewayWireError` (see [`crate::query`]): this one can carry rich +//! internal context (anyhow chains, plugin specifics); the wire one is narrow +//! on purpose since adding variants is a protocol change. Conversion happens +//! at the protocol boundary. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// Content-addressed hash string carried over the wire. Renamed from +/// `Midhash` in the multi-outcome plugin contract migration — the name +/// `Midhash` lied for sha2-256 CIDs (which gateway plugins also emit +/// via `compute_ipfs_cid`). The carrier is generic; the hash family is +/// discriminated by `HashKind` on `HashOutcome`. +/// +/// Shape: base32-lower 'b'-prefixed multibase CIDv1. The specific +/// multihash inside (`0x1000` for midhash256, `0x12` for sha2-256) is +/// determined by the producing plugin's hashing path. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Hash(pub String); + +impl Hash { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for Hash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Pre-hash identifier for a record discovered through a gateway. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct DiscoveryId { + pub upstream_id: String, + pub record_id: String, +} + +/// Metadata-only search result. Distinct from meta-core's persistent record +/// (which is a flat `map[string]string` indexed by midhash); a +/// `DiscoveryRecord` only lives inside the gateway search/response pipeline. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DiscoveryRecord { + pub upstream_id: String, + pub record_id: String, + pub fields: BTreeMap, +} + +/// In-process error returned by `FeederPlugin` methods. +#[derive(Debug, thiserror::Error)] +pub enum GatewayError { + #[error("upstream transient error: {0}")] + Transient(String), + + #[error("upstream permanent error: {0}")] + Permanent(String), + + #[error("rate limited; retry in {retry_after_s}s")] + RateLimited { retry_after_s: u32 }, + + #[error("record not found upstream")] + NotFound, + + #[error("internal: {0}")] + Internal(#[from] anyhow::Error), +} + +/// Byte stream returned by a plugin's `handle_fetch`. +pub type ByteStream = futures::stream::BoxStream<'static, Result>; + +/// Plugin liveness snapshot surfaced via `/health`. Must not call upstream +/// — `health()` is invoked on every status poll. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case", tag = "state")] +pub enum PluginHealth { + Ok, + Degraded { reason: String }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hash_display_and_as_str_match() { + let m = Hash("bafy123".to_string()); + assert_eq!(m.as_str(), "bafy123"); + assert_eq!(format!("{m}"), "bafy123"); + } + + #[test] + fn discovery_record_roundtrips_through_serde_json() { + let r = DiscoveryRecord { + upstream_id: "scihub".to_string(), + record_id: "10.1038/s41586-021-03819-2".to_string(), + fields: BTreeMap::from_iter([ + ("title".to_string(), "AlphaFold".to_string()), + ("year".to_string(), "2021".to_string()), + ]), + }; + let j = serde_json::to_string(&r).expect("serialize"); + let back: DiscoveryRecord = serde_json::from_str(&j).expect("deserialize"); + assert_eq!(r, back); + } + + #[test] + fn plugin_health_serialises_with_tag() { + let ok = serde_json::to_value(PluginHealth::Ok).unwrap(); + assert_eq!(ok, serde_json::json!({"state": "ok"})); + let bad = serde_json::to_value(PluginHealth::Degraded { + reason: "upstream 5xx".into(), + }) + .unwrap(); + assert_eq!( + bad, + serde_json::json!({"state": "degraded", "reason": "upstream 5xx"}) + ); + } +} diff --git a/feeder-plugin/book-feeder/Cargo.toml b/feeder-plugin/book-feeder/Cargo.toml new file mode 100644 index 0000000..dc91c99 --- /dev/null +++ b/feeder-plugin/book-feeder/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "book-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Book feeder sidecar: Project Gutenberg (via Gutendex). The proof-of-concept +# feeder — single upstream, no preview-seeding, no BitTorrent — that validates +# the meta-feeder-sdk + feeder HTTP contract end to end. + +[lib] +name = "book_feeder" +path = "src/lib.rs" + +[[bin]] +name = "book-feeder" +path = "src/main.rs" + +[dependencies] +meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +bytes = "1" +futures = "0.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +base64 = "0.22" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/book-feeder/Dockerfile b/feeder-plugin/book-feeder/Dockerfile new file mode 100644 index 0000000..c2bfdd1 --- /dev/null +++ b/feeder-plugin/book-feeder/Dockerfile @@ -0,0 +1,55 @@ +# syntax=docker/dockerfile:1 +# ============================================================================= +# book-feeder — Project Gutenberg feeder sidecar. Multi-stage build. +# +# The build context must be `packages/meta-gateway/` (the workspace root) so +# the `meta-feeder-sdk` path dependency (../../crates/meta-feeder-sdk) resolves. +# Same BuildKit cache-mount strategy as the gateway Dockerfile: the cargo +# registry/git and `target/` persist across builds. The release binary is +# copied OUT of the target cache mount within the same RUN. +# ============================================================================= + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config \ + libssl-dev \ + ca-certificates \ + git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates/ crates/ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=meta-gateway-target,target=/build/target \ + cargo build --release --bin book-feeder \ + && cp target/release/book-feeder /book-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /book-feeder /usr/local/bin/book-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/book-feeder"] diff --git a/feeder-plugin/book-feeder/src/gutenberg.rs b/feeder-plugin/book-feeder/src/gutenberg.rs new file mode 100644 index 0000000..ca905cf --- /dev/null +++ b/feeder-plugin/book-feeder/src/gutenberg.rs @@ -0,0 +1,961 @@ +//! Project Gutenberg bridge via the [Gutendex](https://gutendex.com) JSON API. +//! +//! - Search: `GET /books?search={q}` → JSON list of book records. +//! - Fetch one: `GET /books/{id}` → single book record with a `formats` map. +//! - Canonical file format for midhash: `application/epub+zip`. Books that +//! don't expose an epub variant fail `compute_outcomes` with a permanent +//! error (operator-visible; very rare in practice). +//! +//! No auth, no API key, no rate-limit configuration. Moved verbatim from the +//! gateway crate's `plugins/gutenberg.rs`; only the import paths changed +//! (`crate::*`/`super::common` → `meta_feeder_sdk::*`, `GatewayPlugin` → +//! `FeederPlugin`). + +use std::collections::{BTreeMap, HashMap}; +use std::path::Path; + +use async_trait::async_trait; +use meta_feeder_sdk::cache::MidhashCache; +use meta_feeder_sdk::common; +#[cfg(test)] +use meta_feeder_sdk::plugin::HashKind; +use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, HashOutcome}; +use meta_feeder_sdk::query::GatewayQuery; +use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth}; +use serde::Deserialize; +use tracing::{debug, warn}; + +/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`] +/// for tests + private mirrors. +const DEFAULT_BASE_URL: &str = "https://gutendex.com"; + +/// Format we treat as the canonical hashable bytes for a Gutenberg record. +const EPUB_FORMAT: &str = "application/epub+zip"; + +/// HTTP timeout for each upstream call. +const HTTP_TIMEOUT_SECS: u64 = 30; + +/// User-Agent string sent on every Gutendex request. +const USER_AGENT: &str = concat!("meta-share/", env!("CARGO_PKG_VERSION"), " (gateway:gutenberg)"); + +/// Gutenberg gateway plugin. Cheap to construct; `configure()` opens the +/// per-plugin redb cache. +pub struct GutenbergPlugin { + http: reqwest::Client, + base_url: String, + cache: Option, +} + +impl GutenbergPlugin { + pub fn new() -> Self { + Self::with_base_url(DEFAULT_BASE_URL.to_string()) + } + + /// Construct a plugin pointing at `base_url` instead of the public + /// Gutendex. Used by tests (against a `wiremock::MockServer`) and by + /// private-mirror deployments. + pub fn with_base_url(base_url: String) -> Self { + let http = common::build_http_client(HTTP_TIMEOUT_SECS, USER_AGENT, None); + Self { + http, + base_url, + cache: None, + } + } + + fn cache(&self) -> Result<&MidhashCache, GatewayError> { + common::require_cache(self.cache.as_ref(), "gutenberg") + } + + /// GET `{base}/books/{id}`; map HTTP/JSON failures to `GatewayError`. + async fn fetch_book(&self, record_id: &str) -> Result { + let url = format!("{}/books/{}", self.base_url.trim_end_matches('/'), record_id); + let resp = self + .http + .get(&url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + common::map_status(&resp)?; + resp.json::() + .await + .map_err(|e| GatewayError::Permanent(format!("decode gutendex book {record_id}: {e}"))) + } + + /// GET arbitrary upstream URL and return the body as bytes. + async fn fetch_bytes(&self, url: &str) -> Result { + common::fetch_bytes(&self.http, url).await + } + + /// Fetch + parse each result's Project Gutenberg RDF/XML metadata for the + /// richer per-edition fields the Gutendex JSON API doesn't carry. Cached + /// aggressively by `record_id`. Failures degrade gracefully. + async fn fetch_bibrec_fields(&self, books: &[GutendexBook]) -> Vec> { + use futures::stream::{self, StreamExt}; + const BIBREC_CONCURRENCY: usize = 6; + let cache = self.cache.clone(); + let http = self.http.clone(); + let ids: Vec = books.iter().map(|b| b.id).collect(); + stream::iter(ids.into_iter()) + .map(|id| { + let cache = cache.clone(); + let http = http.clone(); + async move { + let record_id = id.to_string(); + if let Some(cached) = cache + .as_ref() + .and_then(|c| c.get_bibrec(&record_id).ok().flatten()) + { + return cached; + } + let url = format!("https://www.gutenberg.org/cache/epub/{id}/pg{id}.rdf"); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(8), + http.get(&url).send(), + ) + .await; + let xml = match resp { + Ok(Ok(r)) if r.status().is_success() => r.text().await.ok(), + Ok(Ok(r)) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + id, + status = %r.status(), + "rdf fetch non-2xx" + ); + None + } + Ok(Err(e)) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + id, + error = %e, + "rdf fetch failed" + ); + None + } + Err(_) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + id, + "rdf fetch timed out" + ); + None + } + }; + let Some(xml) = xml else { + return BTreeMap::new(); + }; + let fields = parse_pgterms_rdf(&xml); + if !fields.is_empty() { + if let Some(cache) = cache.as_ref() { + if let Err(e) = cache.put_bibrec(&record_id, &fields) { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + record_id = %record_id, + error = %e, + "bibrec put failed (non-fatal)" + ); + } + } + } + fields + } + }) + .buffered(BIBREC_CONCURRENCY) + .collect() + .await + } + + /// Download every result's cover image (where present), hash, and cache. + /// Returns a `Vec>` of cover CIDs aligned with the input + /// slice — `None` means no cover or download failure. + async fn fetch_cover_cids(&self, books: &[GutendexBook]) -> Vec> { + use futures::stream::{self, StreamExt}; + const COVER_CONCURRENCY: usize = 8; + let cache = self.cache.clone(); + let plugin_http = self.http.clone(); + let to_process: Vec<(usize, String, Option, Option)> = books + .iter() + .enumerate() + .map(|(idx, b)| { + let record_id = b.id.to_string(); + let cover_url = b.formats.get("image/jpeg").cloned(); + let cached_cid = cache + .as_ref() + .and_then(|c| c.get_cover_cid(&record_id).ok().flatten()); + (idx, record_id, cover_url, cached_cid) + }) + .collect(); + stream::iter(to_process.into_iter()) + .map(|(idx, record_id, cover_url, cached_cid)| { + let cache = cache.clone(); + let http = plugin_http.clone(); + async move { + if let Some(cid) = cached_cid { + return Some(cid); + } + let url = cover_url?; + let bytes_res = tokio::time::timeout( + std::time::Duration::from_secs(8), + http.get(&url).send(), + ) + .await; + let resp = match bytes_res { + Ok(Ok(r)) if r.status().is_success() => r, + Ok(Ok(r)) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + book_id = idx, + status = %r.status(), + "cover fetch non-2xx" + ); + return None; + } + Ok(Err(e)) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + book_id = idx, + error = %e, + "cover fetch failed" + ); + return None; + } + Err(_) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + book_id = idx, + "cover fetch timed out" + ); + return None; + } + }; + let bytes = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + book_id = idx, + error = %e, + "cover body read failed" + ); + return None; + } + }; + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + if let Some(cache) = cache.as_ref() { + if let Err(e) = cache.put_blob(&cid, &bytes) { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + cid = %cid, + error = %e, + "cover put_blob failed (non-fatal)" + ); + } + if let Err(e) = cache.put_cover_cid(&record_id, &cid) { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + record_id = %record_id, + error = %e, + "cover put_cover_cid failed (non-fatal)" + ); + } + } + Some(cid) + } + }) + .buffered(COVER_CONCURRENCY) + .collect() + .await + } +} + +impl Default for GutenbergPlugin { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl FeederPlugin for GutenbergPlugin { + fn upstream_id(&self) -> &'static str { + "gutenberg" + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + self.cache = Some(common::open_midhash_cache(cache_dir, "gutenberg")?); + Ok(()) + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + // Layer A early-return: Gutenberg only serves `document` / `book`. + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + let q = query.free_text_or_star(); + let url = format!("{}/books", self.base_url.trim_end_matches('/')); + let resp = self + .http + .get(&url) + .query(&[("search", q)]) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + common::map_status(&resp)?; + let body: GutendexSearch = resp.json().await.map_err(|e| { + GatewayError::Permanent(format!("decode gutendex search response: {e}")) + })?; + debug!( + target: "meta-share::gateway", + upstream = "gutenberg", + query = q, + count = body.results.len(), + "search returned" + ); + let raw: Vec = body.results.into_iter().take(max_results).collect(); + let (cover_cids, bibrecs) = + tokio::join!(self.fetch_cover_cids(&raw), self.fetch_bibrec_fields(&raw),); + Ok(raw + .into_iter() + .zip(cover_cids) + .zip(bibrecs) + .map(|((book, cover_cid), bibrec)| into_discovery_record(book, cover_cid, bibrec)) + .collect()) + } + + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let cache = self.cache()?; + if let Some(hit) = common::cached_outcome(cache, record_id, "gutenberg")? { + return Ok(hit); + } + + let book = self.fetch_book(record_id).await?; + let epub_url = book.formats.get(EPUB_FORMAT).cloned().ok_or_else(|| { + GatewayError::Permanent(format!( + "gutenberg record {record_id} has no {EPUB_FORMAT} format" + )) + })?; + let bytes = self.fetch_bytes(&epub_url).await?; + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + + common::store_midhash(cache, record_id, "gutenberg", &cid); + + let cover_cid = cache.get_cover_cid(record_id).ok().flatten(); + let bibrec = cache + .get_bibrec(record_id) + .ok() + .flatten() + .unwrap_or_default(); + let record = into_discovery_record(book, cover_cid, bibrec); + + Ok(common::single_outcome( + cid, + bytes, + record, + Some("epub".to_string()), + )) + } + + async fn get_blob(&self, cid: &str) -> Option> { + match self.cache.as_ref()?.get_blob(cid) { + Ok(opt) => opt, + Err(e) => { + warn!( + target: "meta-share::gateway", + upstream = "gutenberg", + cid, + error = %e, + "get_blob: cache read failed" + ); + None + } + } + } + + fn health(&self) -> PluginHealth { + if self.cache.is_some() { + PluginHealth::Ok + } else { + PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + } + } + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["document"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &["book"] + } +} + +/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`. +fn into_discovery_record( + book: GutendexBook, + cover_cid: Option, + bibrec: BTreeMap, +) -> DiscoveryRecord { + let GutendexBook { + id, + title, + authors, + subjects, + bookshelves, + languages, + copyright, + media_type, + formats, + download_count, + } = book; + + let record_id = id.to_string(); + let mut fields = BTreeMap::new(); + fields.insert("title".into(), title); + fields.insert("fileType".into(), "document".into()); + fields.insert("contentKind".into(), "book".into()); + fields.insert( + "sourceUrl".into(), + format!("https://www.gutenberg.org/ebooks/{id}"), + ); + // Canonical `id` field. + fields.insert(upstream_id_field("gutenberg"), id.to_string()); + + if !authors.is_empty() { + fields.insert( + "author".into(), + authors + .iter() + .map(|a| a.name.as_str()) + .collect::>() + .join(", "), + ); + } + if let Some(lang) = languages.first() { + fields.insert("language".into(), iso639_1_to_2(lang).to_string()); + } + if !subjects.is_empty() { + fields.insert("subjects".into(), subjects.join(", ")); + } + if !bookshelves.is_empty() { + fields.insert("bookshelves".into(), bookshelves.join(", ")); + } + if let Some(mt) = media_type { + fields.insert("mediaType".into(), mt); + } + if let Some(cid) = cover_cid { + fields.insert("poster".into(), cid); + } + if formats.contains_key(EPUB_FORMAT) { + fields.insert("format".into(), "epub".into()); + } + if let Some(c) = copyright { + fields.insert("publicDomain".into(), (!c).to_string()); + } + let _ = download_count; + fields.insert("fileName".into(), format!("gutenberg-{id}.epub")); + + for (k, v) in bibrec { + fields.entry(k).or_insert(v); + } + + DiscoveryRecord { + upstream_id: "gutenberg".into(), + record_id, + fields, + } +} + +/// Parse a Project Gutenberg `pg{id}.rdf` document for per-edition metadata. +fn parse_pgterms_rdf(xml: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + + let pull = |tag: &str| extract_first_tag_text(xml, tag); + + if let Some(v) = pull("dcterms:issued") { + out.insert("releaseDate".into(), v); + } + if let Some(v) = pull("dcterms:rights") { + out.insert("copyrightStatus".into(), v); + } + if let Some(v) = pull("dcterms:alternative") { + out.insert("alternateTitle".into(), v); + } + if let Some(v) = pull("dcterms:description") { + if let Some(url) = v + .split_whitespace() + .find(|t| t.contains("wikipedia.org") && t.starts_with("http")) + { + out.insert("wikipediaUrl".into(), url.to_string()); + } + out.insert("note".into(), v); + } + if let Some(v) = pull("pgterms:marc508") { + out.insert("credits".into(), v); + } + if let Some(v) = pull("pgterms:marc520") { + out.insert("summary".into(), v); + } + if let Some(v) = pull("pgterms:marc908") { + out.insert("readingLevel".into(), v); + } + if let Some(url) = extract_resource_attr(xml, "pgterms:webpage") { + out.insert("authorWebpage".into(), url); + } + + out +} + +/// Find the first `BODY` and return BODY trimmed. +fn extract_first_tag_text(xml: &str, tag: &str) -> Option { + let open_prefix = format!("<{tag}"); + let close = format!(""); + let open_start = xml.find(&open_prefix)?; + let after_attrs = open_start + open_prefix.len(); + let rest = &xml[after_attrs..]; + let body_start = after_attrs + rest.find('>')? + 1; + let body_end_rel = xml[body_start..].find(&close)?; + let body = &xml[body_start..body_start + body_end_rel]; + let decoded = body + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'"); + let trimmed = decoded.trim().to_string(); + if trimmed.is_empty() { + None + } else { + Some(trimmed) + } +} + +/// Find the first ` Option { + let open_prefix = format!("<{tag} "); + let open_start = xml.find(&open_prefix)?; + let rest = &xml[open_start + open_prefix.len()..]; + let tag_end = rest.find('>')?; + let inside = &rest[..tag_end]; + let needle = "rdf:resource=\""; + let attr_start = inside.find(needle)?; + let after = &inside[attr_start + needle.len()..]; + let url_end = after.find('"')?; + Some(after[..url_end].to_string()) +} + +#[cfg(test)] +mod rdf_parser_tests { + use super::*; + + #[test] + fn extracts_release_date_and_copyright() { + let xml = r#" + + 2008-06-27 + Public domain in the USA. + + "#; + let f = parse_pgterms_rdf(xml); + assert_eq!(f.get("releaseDate").map(String::as_str), Some("2008-06-27")); + assert_eq!( + f.get("copyrightStatus").map(String::as_str), + Some("Public domain in the USA.") + ); + } + + #[test] + fn extracts_credits_summary_reading_level() { + let xml = r#" + Arthur DiBianca and David Widger + "Alice's..." short summary. + Reading ease score: 87.5 (6th grade). Easy to read. + "#; + let f = parse_pgterms_rdf(xml); + assert_eq!( + f.get("credits").map(String::as_str), + Some("Arthur DiBianca and David Widger") + ); + assert!(f.get("summary").unwrap().contains("short summary")); + assert!(f.get("readingLevel").unwrap().contains("87.5")); + } + + #[test] + fn pulls_wikipedia_url_out_of_description() { + let xml = r#" + Wikipedia page about this book: https://en.wikipedia.org/wiki/Alice%27s_Adventures_in_Wonderland + "#; + let f = parse_pgterms_rdf(xml); + assert_eq!( + f.get("wikipediaUrl").map(String::as_str), + Some("https://en.wikipedia.org/wiki/Alice%27s_Adventures_in_Wonderland") + ); + assert!(f.get("note").unwrap().contains("Wikipedia")); + } + + #[test] + fn empty_when_not_an_rdf_doc() { + let f = parse_pgterms_rdf("not rdf"); + assert!(f.is_empty()); + } +} + +/// ISO 639-1 (two-letter) → ISO 639-2/T (three-letter) for the ~20 most common +/// Gutenberg languages. Unknown codes pass through unchanged. +fn iso639_1_to_2(code: &str) -> &str { + match code { + "en" => "eng", + "fr" => "fra", + "de" => "deu", + "es" => "spa", + "it" => "ita", + "pt" => "por", + "nl" => "nld", + "la" => "lat", + "el" => "ell", + "ru" => "rus", + "fi" => "fin", + "sv" => "swe", + "da" => "dan", + "no" => "nor", + "pl" => "pol", + "cs" => "ces", + "hu" => "hun", + "ja" => "jpn", + "zh" => "zho", + "ar" => "ara", + "ko" => "kor", + "tr" => "tur", + "ro" => "ron", + "ca" => "cat", + "is" => "isl", + other => other, + } +} + +// -- Gutendex JSON shapes (subset). We only deserialise the fields we use. + +#[derive(Debug, Deserialize)] +struct GutendexSearch { + results: Vec, +} + +#[derive(Debug, Deserialize)] +struct GutendexBook { + id: u64, + title: String, + #[serde(default)] + authors: Vec, + #[serde(default)] + subjects: Vec, + #[serde(default)] + bookshelves: Vec, + #[serde(default)] + languages: Vec, + #[serde(default)] + copyright: Option, + #[serde(default)] + media_type: Option, + #[serde(default)] + formats: HashMap, + #[serde(default)] + download_count: u64, +} + +#[derive(Debug, Deserialize)] +struct GutendexPerson { + name: String, + // Gutendex also returns `birth_year` / `death_year`, but those are about + // the author, not this book file, so we don't deserialize or surface them + // (serde ignores the unknown JSON keys). See `into_discovery_record`. +} + +#[cfg(test)] +mod tests { + use super::*; + use meta_feeder_sdk::types::PluginHealth; + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn books_json() -> serde_json::Value { + serde_json::json!({ + "count": 2, + "next": null, + "previous": null, + "results": [ + { + "id": 1342, + "title": "Pride and Prejudice", + "authors": [{"name": "Austen, Jane", "birth_year": 1775, "death_year": 1817}], + "languages": ["en"], + "formats": { + "application/epub+zip": "EPUB_URL_PLACEHOLDER", + "text/plain; charset=utf-8": "https://www.gutenberg.org/files/1342/1342-0.txt" + } + }, + { + "id": 84, + "title": "Frankenstein; Or, The Modern Prometheus", + "authors": [{"name": "Shelley, Mary Wollstonecraft", "birth_year": 1797, "death_year": 1851}], + "languages": ["en"], + "formats": { + "application/epub+zip": "EPUB_URL_PLACEHOLDER", + "text/plain; charset=utf-8": "https://www.gutenberg.org/files/84/84-0.txt" + } + } + ] + }) + } + + fn configured_plugin_against(server: &MockServer) -> (GutenbergPlugin, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let mut plugin = GutenbergPlugin::with_base_url(server.uri()); + plugin.configure(dir.path()).expect("configure"); + (plugin, dir) + } + + #[tokio::test] + async fn handle_query_maps_results_to_discovery_records() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books")) + .and(query_param("search", "alice")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 50) + .await + .expect("handle_query"); + assert_eq!(records.len(), 2); + + let r0 = &records[0]; + assert_eq!(r0.upstream_id, "gutenberg"); + assert_eq!(r0.record_id, "1342"); + assert_eq!( + r0.fields.get("title").map(String::as_str), + Some("Pride and Prejudice") + ); + assert_eq!(r0.fields.get("fileType").map(String::as_str), Some("document")); + assert_eq!(r0.fields.get("contentKind").map(String::as_str), Some("book")); + assert_eq!(r0.fields.get("gutenbergid").map(String::as_str), Some("1342")); + assert_eq!( + r0.fields.get("sourceUrl").map(String::as_str), + Some("https://www.gutenberg.org/ebooks/1342") + ); + assert_eq!(r0.fields.get("author").map(String::as_str), Some("Austen, Jane")); + assert_eq!(r0.fields.get("language").map(String::as_str), Some("eng")); + assert_eq!(r0.fields.get("format").map(String::as_str), Some("epub")); + assert_eq!( + r0.fields.get("fileName").map(String::as_str), + Some("gutenberg-1342.epub") + ); + } + + #[tokio::test] + async fn handle_query_truncates_to_max_results() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("alice"), 1) + .await + .expect("ok"); + assert_eq!(records.len(), 1); + } + + #[tokio::test] + async fn compute_outcomes_cache_hit_skips_http() { + let server = MockServer::start().await; + let (plugin, _dir) = configured_plugin_against(&server); + plugin + .cache + .as_ref() + .unwrap() + .put_midhash("1342", "bafyCACHED") + .unwrap(); + + let outcomes = plugin.compute_outcomes("1342").await.expect("cache hit"); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].hash.as_str(), "bafyCACHED"); + } + + #[tokio::test] + async fn compute_outcomes_fetches_epub_and_caches() { + let server = MockServer::start().await; + + let epub_bytes: Vec = b"<>".to_vec(); + let expected_cid = meta_feeder_sdk::hash::compute_ipfs_cid(&epub_bytes); + + let epub_path = "/ebooks/1342.epub.images"; + let book = serde_json::json!({ + "id": 1342, + "title": "Pride and Prejudice", + "authors": [{"name": "Austen, Jane"}], + "languages": ["en"], + "formats": { + "application/epub+zip": format!("{}{epub_path}", server.uri()), + } + }); + Mock::given(method("GET")) + .and(path("/books/1342")) + .respond_with(ResponseTemplate::new(200).set_body_json(book)) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path(epub_path)) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(epub_bytes.clone()) + .insert_header("content-type", "application/epub+zip"), + ) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let outcomes = plugin.compute_outcomes("1342").await.expect("compute"); + assert_eq!(outcomes.len(), 1); + assert_eq!(outcomes[0].hash.as_str(), expected_cid); + assert_eq!(outcomes[0].hash_kind, HashKind::Sha2_256); + assert_eq!( + plugin + .cache + .as_ref() + .unwrap() + .get_midhash("1342") + .unwrap() + .as_deref(), + Some(expected_cid.as_str()) + ); + } + + #[tokio::test] + async fn compute_outcomes_missing_epub_returns_permanent() { + let server = MockServer::start().await; + let book = serde_json::json!({ + "id": 99999, + "title": "Audio-only Record", + "authors": [], + "languages": ["en"], + "formats": { + "text/plain; charset=utf-8": format!("{}/files/99999/99999-0.txt", server.uri()), + } + }); + Mock::given(method("GET")) + .and(path("/books/99999")) + .respond_with(ResponseTemplate::new(200).set_body_json(book)) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .compute_outcomes("99999") + .await + .expect_err("should error"); + match err { + GatewayError::Permanent(msg) => { + assert!(msg.contains("epub"), "msg: {msg}"); + assert!(msg.contains("99999"), "msg: {msg}"); + } + other => panic!("wrong variant: {other:?}"), + } + } + + #[tokio::test] + async fn upstream_404_maps_to_not_found() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books/1")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin.compute_outcomes("1").await.expect_err("404"); + assert!(matches!(err, GatewayError::NotFound)); + } + + #[tokio::test] + async fn upstream_5xx_maps_to_transient() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books")) + .respond_with(ResponseTemplate::new(502)) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .handle_query(&GatewayQuery::from_free_text("x"), 10) + .await + .expect_err("502"); + assert!(matches!(err, GatewayError::Transient(_)), "got: {err:?}"); + } + + #[tokio::test] + async fn upstream_429_maps_to_rate_limited_with_retry_after() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books")) + .respond_with(ResponseTemplate::new(429).insert_header("retry-after", "90")) + .mount(&server) + .await; + + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .handle_query(&GatewayQuery::from_free_text("x"), 10) + .await + .expect_err("429"); + match err { + GatewayError::RateLimited { retry_after_s } => { + assert_eq!(retry_after_s, 90); + } + other => panic!("wrong variant: {other:?}"), + } + } + + #[test] + fn health_reflects_configure_state() { + let plugin = GutenbergPlugin::new(); + match plugin.health() { + PluginHealth::Degraded { reason } => assert!(reason.contains("configure")), + PluginHealth::Ok => panic!("unconfigured plugin should not be Ok"), + } + + let dir = tempfile::tempdir().unwrap(); + let mut plugin = GutenbergPlugin::new(); + plugin.configure(dir.path()).unwrap(); + match plugin.health() { + PluginHealth::Ok => {} + PluginHealth::Degraded { reason } => { + panic!("configured plugin should be Ok, got Degraded: {reason}") + } + } + } +} diff --git a/feeder-plugin/book-feeder/src/lib.rs b/feeder-plugin/book-feeder/src/lib.rs new file mode 100644 index 0000000..1f4c37e --- /dev/null +++ b/feeder-plugin/book-feeder/src/lib.rs @@ -0,0 +1,5 @@ +//! `book-feeder` library surface — exposes the Gutenberg plugin so integration +//! tests (and, later, other binaries) can construct it. The binary +//! (`main.rs`) is a thin wrapper over [`meta_feeder_sdk::serve_feeders`]. + +pub mod gutenberg; diff --git a/feeder-plugin/book-feeder/src/main.rs b/feeder-plugin/book-feeder/src/main.rs new file mode 100644 index 0000000..85c6229 --- /dev/null +++ b/feeder-plugin/book-feeder/src/main.rs @@ -0,0 +1,31 @@ +//! `book-feeder` — Project Gutenberg feeder sidecar. +//! +//! Hosts the [`gutenberg::GutenbergPlugin`] over the feeder HTTP contract via +//! [`meta_feeder_sdk::serve_feeders`]. The gateway core reaches it server-to- +//! server on the internal docker network. +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use book_feeder::gutenberg::GutenbergPlugin; +use meta_feeder_sdk::serve_feeders; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + serve_feeders(vec![Box::new(GutenbergPlugin::new())], state_dir, listen).await +} diff --git a/feeder-plugin/book-feeder/tests/contract.rs b/feeder-plugin/book-feeder/tests/contract.rs new file mode 100644 index 0000000..fd78849 --- /dev/null +++ b/feeder-plugin/book-feeder/tests/contract.rs @@ -0,0 +1,172 @@ +//! End-to-end contract test for the feeder HTTP surface: boots the real +//! `serve_feeders` router (book-feeder hosting GutenbergPlugin) on an ephemeral +//! port, pointed at a wiremock Gutendex, and drives `/manifest`, `/query`, and +//! `/compute` over HTTP. Deterministic — no live Gutendex. + +use book_feeder::gutenberg::GutenbergPlugin; +use meta_feeder_sdk::{ + configure_plugins, hash, router, ComputeRequest, ComputeResponse, GatewayQuery, HashKindDto, + ManifestResponse, QueryRequest, QueryResponse, +}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +/// Spawn the feeder router (Gutenberg pointed at `upstream`) on an ephemeral +/// port; return its base URL. The TempDir is returned so the per-plugin cache +/// outlives the test. +async fn spawn_feeder(upstream: &MockServer) -> (String, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let plugin = GutenbergPlugin::with_base_url(upstream.uri()); + let plugins = + configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure plugins"); + let app = router(plugins, "book-feeder-test".to_string(), dir.path()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral"); + let addr = listener.local_addr().expect("local_addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + (format!("http://{addr}"), dir) +} + +fn books_json() -> serde_json::Value { + serde_json::json!({ + "results": [{ + "id": 1342, + "title": "Pride and Prejudice", + "authors": [{"name": "Austen, Jane"}], + "languages": ["en"], + "formats": { "application/epub+zip": "EPUB_PLACEHOLDER" } + }] + }) +} + +#[tokio::test] +async fn manifest_lists_book_capabilities() { + let upstream = MockServer::start().await; + let (base, _dir) = spawn_feeder(&upstream).await; + + let manifest: ManifestResponse = reqwest::Client::new() + .get(format!("{base}/manifest")) + .send() + .await + .expect("GET /manifest") + .json() + .await + .expect("decode manifest"); + + assert_eq!(manifest.plugins.len(), 1); + let p = &manifest.plugins[0]; + assert_eq!(p.id, "gutenberg"); + assert_eq!(p.served_file_types, vec!["document".to_string()]); + assert_eq!(p.served_content_kinds, vec!["book".to_string()]); +} + +#[tokio::test] +async fn query_returns_gutenberg_records() { + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/books")) + .respond_with(ResponseTemplate::new(200).set_body_json(books_json())) + .mount(&upstream) + .await; + let (base, _dir) = spawn_feeder(&upstream).await; + + let resp: QueryResponse = reqwest::Client::new() + .post(format!("{base}/query")) + .json(&QueryRequest { + upstream_id: "gutenberg".to_string(), + query: GatewayQuery::from_free_text("alice"), + max_results: 50, + }) + .send() + .await + .expect("POST /query") + .json() + .await + .expect("decode query response"); + + assert_eq!(resp.records.len(), 1); + let r = &resp.records[0]; + assert_eq!(r.upstream_id, "gutenberg"); + assert_eq!(r.fields.get("gutenbergid").map(String::as_str), Some("1342")); + assert_eq!(r.fields.get("contentKind").map(String::as_str), Some("book")); +} + +#[tokio::test] +async fn compute_returns_outcome_with_inline_epub_bytes() { + let upstream = MockServer::start().await; + let epub_bytes = b"<>".to_vec(); + let expected_cid = hash::compute_ipfs_cid(&epub_bytes); + let epub_path = "/ebooks/1342.epub"; + + let book = serde_json::json!({ + "id": 1342, + "title": "Pride and Prejudice", + "authors": [{"name": "Austen, Jane"}], + "languages": ["en"], + "formats": { "application/epub+zip": format!("{}{epub_path}", upstream.uri()) } + }); + Mock::given(method("GET")) + .and(path("/books/1342")) + .respond_with(ResponseTemplate::new(200).set_body_json(book)) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path(epub_path)) + .respond_with(ResponseTemplate::new(200).set_body_bytes(epub_bytes.clone())) + .mount(&upstream) + .await; + + let (base, _dir) = spawn_feeder(&upstream).await; + + let resp: ComputeResponse = reqwest::Client::new() + .post(format!("{base}/compute")) + .json(&ComputeRequest { + upstream_id: "gutenberg".to_string(), + record_id: "1342".to_string(), + }) + .send() + .await + .expect("POST /compute") + .json() + .await + .expect("decode compute response"); + + assert_eq!(resp.outcomes.len(), 1); + let o = &resp.outcomes[0]; + assert_eq!(o.hash, expected_cid); + assert_eq!(o.hash_kind, HashKindDto::Sha2_256); + assert_eq!(o.file_extension.as_deref(), Some("epub")); + // Bytes ride inline base64 on the v1 contract — decode + verify they hash + // back to the advertised CID (the core's auto-store will do exactly this). + let b64 = o.bytes_b64.as_ref().expect("compute returned inline bytes"); + use base64::Engine as _; + let decoded = base64::engine::general_purpose::STANDARD + .decode(b64) + .expect("valid base64"); + assert_eq!(decoded, epub_bytes); + assert_eq!(hash::compute_ipfs_cid(&decoded), expected_cid); +} + +#[tokio::test] +async fn query_unknown_upstream_is_404() { + let upstream = MockServer::start().await; + let (base, _dir) = spawn_feeder(&upstream).await; + + let status = reqwest::Client::new() + .post(format!("{base}/query")) + .json(&QueryRequest { + upstream_id: "torznab".to_string(), + query: GatewayQuery::from_free_text("x"), + max_results: 10, + }) + .send() + .await + .expect("POST /query") + .status(); + + assert_eq!(status, reqwest::StatusCode::NOT_FOUND); +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]