diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1de5659 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..e6a5a6f --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2177 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tribler-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "bytes", + "chrono", + "futures", + "meta-feeder-sdk", + "regex", + "reqwest", + "serde", + "serde_json", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[patch.unused]] +name = "core2" +version = "0.4.0" +source = "git+https://github.com/bbqsrc/core2?rev=545e84b#545e84bcb0f235b12e21351e0c69767958efe2a7" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..df184bc --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,18 @@ +[workspace] +resolver = "2" +members = ["crates/*", "feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-tribler" + +# --------------------------------------------------------------------------- +# Safety net for the yanked transitive `core2` dep, kept identical to the +# meta-gateway / meta-feeder-torrent workspaces. Harmless (an unused-patch +# warning) if the dep graph doesn't pull it; required if it does. Remove once +# multihash-codetable drops the core2 dependency upstream. +# --------------------------------------------------------------------------- +[patch.crates-io] +core2 = { git = "https://github.com/bbqsrc/core2", rev = "545e84b" } diff --git a/crates/meta-feeder-sdk/Cargo.toml b/crates/meta-feeder-sdk/Cargo.toml new file mode 100644 index 0000000..bbe98ff --- /dev/null +++ b/crates/meta-feeder-sdk/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "meta-feeder-sdk" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# Source-agnostic foundation shared by the gateway core and every feeder +# sidecar binary: the `FeederPlugin` trait, the content-addressing hash +# families, the per-plugin redb cache, the shared HTTP/cache helpers, the +# structured-query evaluator, and the axum `serve_feeders` harness that wraps +# any set of `FeederPlugin`s in the feeder HTTP contract. +# +# Deliberately libp2p-free / blockstore-free: a feeder finds + fetches bytes; +# the gateway core owns hashing-into-the-blockstore, the meta-core store-back, +# and the libp2p wire. Keep heavy deps (librqbit, beetswap, cid, libp2p) OUT +# of here so feeders that don't need them don't pay for them. + +[dependencies] +# Async runtime + HTTP server (the serve_feeders harness) +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +hyper = { version = "1", features = ["full"] } +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" +base64 = "0.22" + +# Utilities +futures = "0.3" +async-trait = "0.1" +anyhow = "1" +thiserror = "1" +tracing = "0.1" +sha2 = "0.10" +bytes = "1" +# Title-tag parsing (shared filename_meta — season/episode/quality/codec/languages). +regex = "1" + +# Per-plugin midhash/blob cache. Schema is identical to meta-share v1 — see +# cache.rs. Pure-Rust single-process embedded store. +redb = "2" + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/crates/meta-feeder-sdk/src/cache.rs b/crates/meta-feeder-sdk/src/cache.rs new file mode 100644 index 0000000..f22e99e --- /dev/null +++ b/crates/meta-feeder-sdk/src/cache.rs @@ -0,0 +1,337 @@ +//! Per-plugin midhash cache backed by redb. +//! +//! Each plugin gets its own redb file at +//! `/gateway//cache.redb`. Keys are upstream +//! `record_id`s (DOI for sci-hub, book id for gutenberg, …); values are the +//! `Midhash` strings the plugin computed by fetching + hashing the upstream +//! file. +//! +//! The schema is identical to meta-share v1's so old `.redb` files copy +//! across. redb is synchronous and sub-millisecond for small ops, so calls +//! run inline from async handlers without `spawn_blocking`. + +use std::path::Path; +use std::sync::Arc; + +use redb::{Database, ReadableTableMetadata, TableDefinition}; + +pub const CACHE_FILENAME: &str = "cache.redb"; + +const MIDHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("midhash"); +const BLOBS_TABLE: TableDefinition<'_, &str, &[u8]> = TableDefinition::new("blobs"); +const COVER_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("cover_cid"); +const BIBREC_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("bibrec"); +const PREVIEW_CID_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("preview_cid"); +/// Full-file sha2-256 IPFS cid for a previously-fetched torrent payload +/// (torznab's bt-fetch path). Distinct from `MIDHASH_TABLE` because the +/// hash families differ: `midhash` is the synthetic midhash256-from-infohash +/// fallback; `fullhash` records "we actually downloaded the bytes and this is +/// the IPFS CIDv1 over them." +const FULLHASH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("fullhash"); +/// Enumerated torrent file list (JSON `Vec`) keyed by the +/// torrent's `record_id` (infohash). +const FILELIST_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("filelist"); +/// Cached TMDB *search* results keyed by a stable lookup key. The value is the +/// JSON encoding of the top `TmdbHit`, or the literal `"null"` negative-cache +/// sentinel. +const TMDB_SEARCH_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_search"); +/// Cached TMDB `GET /3/tv/{id}` structural details keyed by `tmdbid`. +const TMDB_TVDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_tvdetails"); +/// Cached TMDB `GET /3/{tv,movie}/{id}/external_ids` keyed by `tmdbid`. +const TMDB_EXTIDS_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("tmdb_extids"); +/// Cached TMDB `GET /3/movie/{id}` details keyed by `tmdbid`. +const TMDB_MOVIEDETAILS_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_moviedetails"); +/// Cached **ranked** TMDB `search/multi` anchors keyed by the normalized +/// free-text query. +const TMDB_PRINCIPAL_TOPN_TABLE: TableDefinition<'_, &str, &str> = + TableDefinition::new("tmdb_principal_topn"); +/// Subtitle linkage discovered at search-enrich time, keyed by the torrent's +/// `record_id` (infohash). Value is the JSON encoding of a `Vec`. +const SUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("subtitles"); +/// OpenSubtitles search results keyed by `"\x01"`. Value is the +/// resolved subtitle's cid, or the literal `"null"` negative-cache sentinel. +const OPENSUBTITLES_TABLE: TableDefinition<'_, &str, &str> = TableDefinition::new("opensubtitles"); + +/// Per-plugin midhash cache. Cheap to clone (the inner `Database` is shared +/// via `Arc`). +#[derive(Clone)] +pub struct MidhashCache { + db: Arc, +} + +/// Generate the standard `get`/`put` accessor pair for a `&str → &str` redb +/// table. +macro_rules! str_table_accessors { + ($(#[$gmeta:meta])* $get:ident, $(#[$pmeta:meta])* $put:ident, $table:ident $(,)?) => { + $(#[$gmeta])* + pub fn $get(&self, key: &str) -> Result, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table($table)?; + Ok(table.get(key)?.map(|v| v.value().to_string())) + } + + $(#[$pmeta])* + pub fn $put(&self, key: &str, value: &str) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table($table)?; + table.insert(key, value)?; + } + tx.commit()?; + Ok(()) + } + }; +} + +impl MidhashCache { + pub fn open(cache_dir: &Path) -> Result { + let path = cache_dir.join(CACHE_FILENAME); + let db = Database::create(&path)?; + { + let tx = db.begin_write()?; + tx.open_table(MIDHASH_TABLE)?; + tx.open_table(BLOBS_TABLE)?; + tx.open_table(COVER_CID_TABLE)?; + tx.open_table(BIBREC_TABLE)?; + tx.open_table(PREVIEW_CID_TABLE)?; + tx.open_table(FULLHASH_TABLE)?; + tx.open_table(FILELIST_TABLE)?; + tx.open_table(TMDB_SEARCH_TABLE)?; + tx.open_table(TMDB_TVDETAILS_TABLE)?; + tx.open_table(TMDB_EXTIDS_TABLE)?; + tx.open_table(TMDB_MOVIEDETAILS_TABLE)?; + tx.open_table(TMDB_PRINCIPAL_TOPN_TABLE)?; + tx.open_table(SUBTITLES_TABLE)?; + tx.open_table(OPENSUBTITLES_TABLE)?; + tx.commit()?; + } + Ok(Self { db: Arc::new(db) }) + } + + str_table_accessors!(get_midhash, put_midhash, MIDHASH_TABLE); + + pub fn entry_count(&self) -> Result { + let tx = self.db.begin_read()?; + let table = tx.open_table(MIDHASH_TABLE)?; + Ok(table.len()?) + } + + pub fn get_blob(&self, cid: &str) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BLOBS_TABLE)?; + Ok(table.get(cid)?.map(|v| v.value().to_vec())) + } + + pub fn put_blob(&self, cid: &str, bytes: &[u8]) -> Result<(), redb::Error> { + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BLOBS_TABLE)?; + table.insert(cid, bytes)?; + } + tx.commit()?; + Ok(()) + } + + str_table_accessors!(get_cover_cid, put_cover_cid, COVER_CID_TABLE); + + str_table_accessors!( + /// Read a previously-stored full-file IPFS cid for `record_id`. + get_fullhash, + /// Record the IPFS cid produced by a successful full fetch. + put_fullhash, + FULLHASH_TABLE + ); + + str_table_accessors!( + /// Read the cached torrent file list (JSON) for `record_id`. + get_filelist, + /// Cache the enumerated torrent file list (JSON `Vec`). + put_filelist, + FILELIST_TABLE + ); + + str_table_accessors!( + /// Read the cached subtitle linkage (JSON `Vec`). + get_subtitles, + /// Cache the discovered subtitle linkage (JSON `Vec`). + put_subtitles, + SUBTITLES_TABLE + ); + + str_table_accessors!( + /// Read a cached OpenSubtitles lookup by `"\x01"`. + get_opensubtitles, + /// Cache an OpenSubtitles lookup result (cid or `"null"` sentinel). + put_opensubtitles, + OPENSUBTITLES_TABLE + ); + + str_table_accessors!(get_preview_cid, put_preview_cid, PREVIEW_CID_TABLE); + + str_table_accessors!( + /// Read a cached TMDB search result by lookup key. + get_tmdb_search, + /// Cache a TMDB search result under `key`. + put_tmdb_search, + TMDB_SEARCH_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB TV-details JSON by `tmdbid`. + get_tmdb_tvdetails, + /// Cache TMDB TV-details JSON under `tmdbid`. + put_tmdb_tvdetails, + TMDB_TVDETAILS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB external-ids JSON by `tmdbid`. + get_tmdb_extids, + /// Cache TMDB external-ids JSON under `tmdbid`. + put_tmdb_extids, + TMDB_EXTIDS_TABLE + ); + + str_table_accessors!( + /// Read cached TMDB movie-details JSON by `tmdbid`. + get_tmdb_moviedetails, + /// Cache TMDB movie-details JSON under `tmdbid`. + put_tmdb_moviedetails, + TMDB_MOVIEDETAILS_TABLE + ); + + str_table_accessors!( + /// Read the cached ranked anchor list by normalized query key. + get_tmdb_principal_topn, + /// Cache the ranked anchor list under the normalized query key. + put_tmdb_principal_topn, + TMDB_PRINCIPAL_TOPN_TABLE + ); + + pub fn get_bibrec( + &self, + record_id: &str, + ) -> Result>, redb::Error> { + let tx = self.db.begin_read()?; + let table = tx.open_table(BIBREC_TABLE)?; + let Some(v) = table.get(record_id)? else { + return Ok(None); + }; + let s = v.value(); + Ok(serde_json::from_str(s).ok()) + } + + pub fn put_bibrec( + &self, + record_id: &str, + fields: &std::collections::BTreeMap, + ) -> Result<(), redb::Error> { + let json = serde_json::to_string(fields).unwrap_or_else(|_| "{}".to_string()); + let tx = self.db.begin_write()?; + { + let mut table = tx.open_table(BIBREC_TABLE)?; + table.insert(record_id, json.as_str())?; + } + tx.commit()?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_cache_dir() -> tempfile::TempDir { + tempfile::tempdir().expect("tempdir") + } + + #[test] + fn put_then_get_returns_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_midhash("10.1038/x").unwrap(), None); + cache.put_midhash("10.1038/x", "bafyXYZ").expect("put"); + assert_eq!( + cache.get_midhash("10.1038/x").unwrap().as_deref(), + Some("bafyXYZ") + ); + } + + #[test] + fn put_overwrites_existing_value() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + cache.put_midhash("k", "v1").unwrap(); + cache.put_midhash("k", "v2").unwrap(); + assert_eq!(cache.get_midhash("k").unwrap().as_deref(), Some("v2")); + } + + #[test] + fn values_persist_across_reopens() { + let dir = tmp_cache_dir(); + { + let cache = MidhashCache::open(dir.path()).expect("open 1"); + cache.put_midhash("k", "persistent").unwrap(); + } + let cache = MidhashCache::open(dir.path()).expect("open 2"); + assert_eq!( + cache.get_midhash("k").unwrap().as_deref(), + Some("persistent") + ); + } + + #[test] + fn tmdb_search_cache_distinguishes_miss_from_absent() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_search("m\u{1}naruto\u{1}").unwrap(), None); + cache.put_tmdb_search("m\u{1}naruto\u{1}", "null").unwrap(); + assert_eq!( + cache + .get_tmdb_search("m\u{1}naruto\u{1}") + .unwrap() + .as_deref(), + Some("null") + ); + cache + .put_tmdb_search("t\u{1}spy x family\u{1}2019", "{\"tmdbid\":120089}") + .unwrap(); + drop(cache); + let cache = MidhashCache::open(dir.path()).expect("reopen"); + assert_eq!( + cache + .get_tmdb_search("t\u{1}spy x family\u{1}2019") + .unwrap() + .as_deref(), + Some("{\"tmdbid\":120089}") + ); + } + + #[test] + fn tmdb_tvdetails_cache_roundtrips() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.get_tmdb_tvdetails("120089").unwrap(), None); + cache + .put_tmdb_tvdetails("120089", "{\"number_of_seasons\":2}") + .unwrap(); + assert_eq!( + cache.get_tmdb_tvdetails("120089").unwrap().as_deref(), + Some("{\"number_of_seasons\":2}") + ); + } + + #[test] + fn entry_count_reflects_inserts() { + let dir = tmp_cache_dir(); + let cache = MidhashCache::open(dir.path()).expect("open"); + assert_eq!(cache.entry_count().unwrap(), 0); + cache.put_midhash("a", "x").unwrap(); + cache.put_midhash("b", "y").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + cache.put_midhash("a", "x2").unwrap(); + assert_eq!(cache.entry_count().unwrap(), 2); + } +} diff --git a/crates/meta-feeder-sdk/src/common.rs b/crates/meta-feeder-sdk/src/common.rs new file mode 100644 index 0000000..c7a09f5 --- /dev/null +++ b/crates/meta-feeder-sdk/src/common.rs @@ -0,0 +1,288 @@ +//! Shared helpers for single-outcome source plugins (`arxiv`, `gutenberg`, +//! `pubmed`, `wikicommons`, `giphy`, `scihub`). +//! +//! These plugins are structurally near-identical: build a reqwest client, map +//! HTTP status → [`GatewayError`], and front a per-record midhash cache whose +//! hit/put boilerplate is the same everywhere. Only the duplicated scaffolding +//! lives here; the genuinely-different bodies stay in the plugin modules. + +use std::path::Path; +use std::time::Duration; + +use tracing::{debug, warn}; + +use crate::cache::MidhashCache; +use crate::plugin::{ConfigError, HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, GatewayError, Hash}; + +/// Build the shared reqwest client every source plugin uses. +pub fn build_http_client( + timeout_secs: u64, + user_agent: &str, + redirect: Option, +) -> reqwest::Client { + let mut builder = reqwest::Client::builder() + .timeout(Duration::from_secs(timeout_secs)) + .user_agent(user_agent); + if let Some(policy) = redirect { + builder = builder.redirect(policy); + } + builder + .build() + .expect("reqwest client build is infallible with rustls-tls") +} + +/// Map a non-success HTTP status to a [`GatewayError`]. +/// +/// - `404` → [`GatewayError::NotFound`] +/// - `429` → [`GatewayError::RateLimited`] (honours `Retry-After`, default 60s) +/// - `5xx` → [`GatewayError::Transient`] +/// - anything else → [`GatewayError::Permanent`] +pub fn map_status(resp: &reqwest::Response) -> Result<(), GatewayError> { + let status = resp.status(); + if status.is_success() { + return Ok(()); + } + let code = status.as_u16(); + if code == 404 { + return Err(GatewayError::NotFound); + } + if code == 429 { + let retry_after = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(60); + return Err(GatewayError::RateLimited { + retry_after_s: retry_after, + }); + } + if status.is_server_error() { + return Err(GatewayError::Transient(format!( + "upstream {} returned {status}", + resp.url() + ))); + } + Err(GatewayError::Permanent(format!( + "upstream {} returned {status}", + resp.url() + ))) +} + +/// Open the per-plugin redb midhash cache, wrapping the error with the plugin +/// name for a clean startup diagnostic. +pub fn open_midhash_cache( + cache_dir: &Path, + plugin: &'static str, +) -> Result { + MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other { + plugin, + source: anyhow::anyhow!("open redb cache: {e}"), + }) +} + +/// Cache-hit fast path for single-outcome plugins. +/// +/// On hit, returns `Some(vec![sparse outcome])` — hash only, no bytes/record — +/// so the core's `(_, None)` branch skips re-storing content. `Ok(None)` is a +/// cache miss; the caller proceeds to fetch + hash. +pub fn cached_outcome( + cache: &MidhashCache, + record_id: &str, + upstream_id: &'static str, +) -> Result>, GatewayError> { + let Some(cached) = cache + .get_midhash(record_id) + .map_err(|e| GatewayError::Internal(anyhow::anyhow!("cache get: {e}")))? + else { + return Ok(None); + }; + debug!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + "hash cache hit" + ); + Ok(Some(vec![HashOutcome { + hash: Hash(cached), + hash_kind: HashKind::Sha2_256, + bytes: None, + record: None, + file_extension: None, + }])) +} + +/// Persist a freshly-computed `(record_id → cid)` mapping. A write failure is +/// non-fatal, so it is logged and swallowed. +pub fn store_midhash(cache: &MidhashCache, record_id: &str, upstream_id: &'static str, cid: &str) { + if let Err(e) = cache.put_midhash(record_id, cid) { + warn!( + target: "meta-share::gateway", + upstream = upstream_id, + record_id, + error = %e, + "hash cache put failed (non-fatal)" + ); + } +} + +/// Resolve the per-plugin cache handle, or produce the standard "not +/// configured" internal error. +pub fn require_cache<'a>( + cache: Option<&'a MidhashCache>, + plugin: &str, +) -> Result<&'a MidhashCache, GatewayError> { + cache.ok_or_else(|| { + GatewayError::Internal(anyhow::anyhow!( + "{plugin} plugin not configured (configure() never called)" + )) + }) +} + +/// Build the single-element outcome vec a single-outcome plugin returns on a +/// fresh (cache-miss) compute: the full bytes + record, a Sha2_256 IPFS CID, +/// and an optional file extension. +pub fn single_outcome( + cid: String, + bytes: bytes::Bytes, + record: DiscoveryRecord, + file_extension: Option, +) -> Vec { + vec![HashOutcome { + hash: Hash(cid), + hash_kind: HashKind::Sha2_256, + bytes: Some(bytes), + record: Some(record), + file_extension, + }] +} + +/// Percent-encode a query-string value using the unreserved set +/// (RFC 3986 `A-Za-z0-9-_.~`) plus `+` for space. +pub fn urlencode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.as_bytes() { + match *b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(*b as char); + } + b' ' => out.push('+'), + other => out.push_str(&format!("%{:02X}", other)), + } + } + out +} + +/// GET `url` and return the body bytes, mapping transport + status errors to +/// [`GatewayError`]. +pub async fn fetch_bytes(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.bytes() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Like [`fetch_bytes`] but decodes the body as UTF-8 text. +pub async fn fetch_text(http: &reqwest::Client, url: &str) -> Result { + let resp = http + .get(url) + .send() + .await + .map_err(|e| GatewayError::Transient(format!("GET {url}: {e}")))?; + map_status(&resp)?; + resp.text() + .await + .map_err(|e| GatewayError::Transient(format!("read body {url}: {e}"))) +} + +/// Decode a v1 BitTorrent infohash string into its raw 20 bytes. +/// +/// Accepts 40 hex chars (case-insensitive) or 32 base32 chars (RFC 4648, +/// case-insensitive). Returns `None` for anything else. +pub fn decode_infohash(s: &str) -> Option<[u8; 20]> { + let bytes = s.as_bytes(); + let raw: Vec = if bytes.len() == 40 && bytes.iter().all(u8::is_ascii_hexdigit) { + let mut out = Vec::with_capacity(20); + for pair in bytes.chunks(2) { + out.push((hex_nibble(pair[0])? << 4) | hex_nibble(pair[1])?); + } + out + } else if bytes.len() == 32 && bytes.iter().all(u8::is_ascii_alphanumeric) { + decode_base32_20(bytes)? + } else { + return None; + }; + raw.try_into().ok() +} + +fn hex_nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Decode 32 RFC-4648 base32 chars (case-insensitive, no padding) into the 20 +/// bytes they encode. +fn decode_base32_20(chars: &[u8]) -> Option> { + let mut out = Vec::with_capacity(20); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &c in chars { + let val = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a', + b'2'..=b'7' => c - b'2' + 26, + _ => return None, + } as u64; + buffer = (buffer << 5) | val; + bits += 5; + if bits >= 8 { + bits -= 8; + out.push((buffer >> bits) as u8); + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decode_infohash_hex_roundtrips() { + let hex = "0123456789abcdef0123456789abcdef01234567"; + let raw = decode_infohash(hex).expect("hex decodes"); + assert_eq!(raw[0], 0x01); + assert_eq!(raw[19], 0x67); + } + + #[test] + fn decode_infohash_is_case_insensitive_hex() { + let lower = decode_infohash("aabbccddeeff00112233445566778899aabbccdd"); + let upper = decode_infohash("AABBCCDDEEFF00112233445566778899AABBCCDD"); + assert!(lower.is_some()); + assert_eq!(lower, upper); + } + + #[test] + fn decode_infohash_accepts_base32() { + let raw = decode_infohash("MFRGGZDFMZTWQ2LKNNWG23TPOBYXE43U"); + assert!(raw.is_some()); + } + + #[test] + fn decode_infohash_rejects_wrong_shape() { + assert!(decode_infohash("tooshort").is_none()); + assert!(decode_infohash("zz23456789abcdef0123456789abcdef01234567").is_none()); + assert!(decode_infohash("").is_none()); + } +} diff --git a/crates/meta-feeder-sdk/src/config.rs b/crates/meta-feeder-sdk/src/config.rs new file mode 100644 index 0000000..c4af5d6 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config.rs @@ -0,0 +1,312 @@ +//! Self-describing per-plugin configuration: schema, redaction, merge, and the +//! generic schema-driven HTML page the feeder serves at `/config`. +//! +//! The point of this module is that **the gateway carries no per-plugin config +//! knowledge** and **the SDK carries no per-plugin field names**. A plugin +//! declares its fields via [`FeederPlugin::config_schema`](crate::FeederPlugin::config_schema); +//! everything here (redaction of secrets, merge-on-save, and the rendered form) +//! is driven entirely by that schema. Add a feeder → it shows up with a working +//! config form and zero UI/gateway changes. +//! +//! ## Persistence & precedence +//! +//! Config lives at `cache_dir/config.json` (the per-plugin dir handed to +//! `configure()`). The plugin reads **file-or-env** at startup: env is the +//! first-boot seed and stays authoritative until the operator saves through the +//! UI, at which point `config.json` exists and wins. There is no runtime reload +//! — a save returns `restart_required: true` (mirrors the gateway's +//! `gateway-config.json` posture). +//! +//! ## Secrets +//! +//! Fields of kind [`FieldKind::Secret`] are **write-only**: never returned to +//! the page (redacted to a `_set: bool`), and a blank incoming value on +//! save means "keep the stored secret". + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +/// A plugin's full config surface — an ordered list of fields. +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ConfigSchema { + pub fields: Vec, +} + +impl ConfigSchema { + /// Convenience: `true` when the plugin declares no configurable fields (the + /// page then renders a "no configuration required" notice). + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } +} + +/// One configurable field. `record_array` fields nest `fields` for the +/// per-record sub-schema (e.g. torznab's `indexers`). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ConfigField { + /// JSON key in the config object. + pub key: String, + /// Human label for the form. + pub label: String, + #[serde(rename = "type")] + pub kind: FieldKind, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub help: Option, + #[serde(default)] + pub required: bool, + /// Sub-fields for [`FieldKind::RecordArray`]. The FIRST sub-field is treated + /// as the record identifier for secret back-fill on merge (e.g. `name`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fields: Option>, +} + +impl ConfigField { + pub fn text(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Text) + } + pub fn secret(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Secret) + } + pub fn bool(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Bool) + } + pub fn number(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::Number) + } + pub fn list(key: &str, label: &str) -> Self { + Self::new(key, label, FieldKind::List) + } + pub fn record_array(key: &str, label: &str, fields: Vec) -> Self { + let mut f = Self::new(key, label, FieldKind::RecordArray); + f.fields = Some(fields); + f + } + fn new(key: &str, label: &str, kind: FieldKind) -> Self { + Self { + key: key.to_string(), + label: label.to_string(), + kind, + help: None, + required: false, + fields: None, + } + } + pub fn with_help(mut self, help: &str) -> Self { + self.help = Some(help.to_string()); + self + } + pub fn required(mut self) -> Self { + self.required = true; + self + } +} + +/// Field widget / value type. Drives both rendering and the secret/merge rules. +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FieldKind { + /// Free text → JSON string. + Text, + /// Write-only secret → JSON string; redacted on read, "blank keeps" on save. + Secret, + /// Checkbox → JSON bool. + Bool, + /// Numeric input → JSON number. + Number, + /// Comma/line list → JSON array of strings. + List, + /// Repeatable record → JSON array of objects keyed by nested `fields`. + RecordArray, +} + +fn is_blank(v: Option<&Value>) -> bool { + match v { + None | Some(Value::Null) => true, + Some(Value::String(s)) => s.is_empty(), + _ => false, + } +} + +/// Produce a UI-safe view of `values`: secret fields are dropped and replaced +/// with a `_set` boolean; everything else passes through. Recurses into +/// `record_array` records for secret sub-fields. +pub fn redact(values: &Value, schema: &ConfigSchema) -> Value { + let src = values.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&src, &schema.fields)) +} + +fn redact_object(src: &Map, fields: &[ConfigField]) -> Map { + let mut out = Map::new(); + for f in fields { + match f.kind { + FieldKind::Secret => { + let set = !is_blank(src.get(&f.key)); + out.insert(format!("{}_set", f.key), Value::Bool(set)); + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let arr = src + .get(&f.key) + .and_then(Value::as_array) + .map(|recs| { + recs.iter() + .map(|r| { + let m = r.as_object().cloned().unwrap_or_default(); + Value::Object(redact_object(&m, subs)) + }) + .collect::>() + }) + .unwrap_or_default(); + out.insert(f.key.clone(), Value::Array(arr)); + } + _ => { + if let Some(v) = src.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// Merge `incoming` (from the UI) onto `base` (the stored/effective config), +/// schema-aware. Returns the full, UNREDACTED object to persist: +/// - secret: blank/absent incoming keeps `base`; non-blank overwrites. +/// - text/number/bool/list: present in incoming overwrites; absent keeps base. +/// - record_array: incoming array replaces base, but blank secret sub-fields are +/// back-filled from the base record matched by the first sub-field (identifier). +pub fn merge(base: &Value, incoming: &Value, schema: &ConfigSchema) -> Value { + let base_obj = base.as_object().cloned().unwrap_or_default(); + let inc_obj = incoming.as_object().cloned().unwrap_or_default(); + Value::Object(merge_object(&base_obj, &inc_obj, &schema.fields)) +} + +fn merge_object( + base: &Map, + inc: &Map, + fields: &[ConfigField], +) -> Map { + let mut out = base.clone(); + for f in fields { + match f.kind { + FieldKind::Secret => { + if !is_blank(inc.get(&f.key)) { + out.insert(f.key.clone(), inc.get(&f.key).cloned().unwrap()); + } + // blank/absent → keep base (no-op) + } + FieldKind::RecordArray => { + let subs = f.fields.as_deref().unwrap_or(&[]); + let id_key = subs.first().map(|s| s.key.clone()); + let base_recs = base.get(&f.key).and_then(Value::as_array).cloned(); + let inc_recs = match inc.get(&f.key).and_then(Value::as_array) { + Some(r) => r.clone(), + None => continue, // absent → keep base array untouched + }; + let merged: Vec = inc_recs + .iter() + .map(|rec| { + let rec_obj = rec.as_object().cloned().unwrap_or_default(); + // Find the matching base record by identifier sub-field. + let base_match = match (&id_key, base_recs.as_ref()) { + (Some(idk), Some(brs)) => brs + .iter() + .find(|br| br.get(idk) == rec_obj.get(idk)) + .and_then(Value::as_object) + .cloned() + .unwrap_or_default(), + _ => Map::new(), + }; + Value::Object(merge_object(&base_match, &rec_obj, subs)) + }) + .collect(); + out.insert(f.key.clone(), Value::Array(merged)); + } + _ => { + if let Some(v) = inc.get(&f.key) { + out.insert(f.key.clone(), v.clone()); + } + } + } + } + out +} + +/// The generic, schema-driven config page served at `GET /config`. Self-contained +/// (inline CSS + JS, no external assets) so it loads identically when hit +/// directly on the feeder OR reverse-proxied through the gateway. All API calls +/// are relative to `location.pathname`, so the proxy mount prefix is irrelevant. +pub const CONFIG_PAGE_HTML: &str = include_str!("config_page.html"); + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn torznab_like() -> ConfigSchema { + ConfigSchema { + fields: vec![ + ConfigField::secret("tmdb_token", "TMDB token"), + ConfigField::bool("resolve_torrent_files", "Resolve .torrent"), + ConfigField::record_array( + "indexers", + "Indexers", + vec![ + ConfigField::text("name", "Name"), + ConfigField::secret("api_key", "API key"), + ], + ), + ], + } + } + + #[test] + fn redact_hides_secrets_and_flags_set() { + let v = json!({ + "tmdb_token": "abc", + "resolve_torrent_files": true, + "indexers": [{"name":"prowlarr","api_key":"k"}] + }); + let r = redact(&v, &torznab_like()); + assert_eq!(r["tmdb_token_set"], json!(true)); + assert!(r.get("tmdb_token").is_none()); + assert_eq!(r["resolve_torrent_files"], json!(true)); + assert_eq!(r["indexers"][0]["name"], json!("prowlarr")); + assert_eq!(r["indexers"][0]["api_key_set"], json!(true)); + assert!(r["indexers"][0].get("api_key").is_none()); + } + + #[test] + fn merge_blank_secret_keeps_base() { + let base = json!({"tmdb_token": "keepme"}); + let inc = json!({"tmdb_token": ""}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("keepme")); + } + + #[test] + fn merge_nonblank_secret_overwrites() { + let base = json!({"tmdb_token": "old"}); + let inc = json!({"tmdb_token": "new"}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["tmdb_token"], json!("new")); + } + + #[test] + fn merge_record_array_backfills_blank_secret_by_identifier() { + let base = json!({"indexers":[{"name":"prowlarr","api_key":"secret"}]}); + // UI re-submits the indexer with a blank api_key → keep the stored one. + let inc = json!({"indexers":[{"name":"prowlarr","api_key":""}]}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["indexers"][0]["api_key"], json!("secret")); + assert_eq!(m["indexers"][0]["name"], json!("prowlarr")); + } + + #[test] + fn merge_bool_overwrites_when_present() { + let base = json!({"resolve_torrent_files": true}); + let inc = json!({"resolve_torrent_files": false}); + let m = merge(&base, &inc, &torznab_like()); + assert_eq!(m["resolve_torrent_files"], json!(false)); + } +} diff --git a/crates/meta-feeder-sdk/src/config_page.html b/crates/meta-feeder-sdk/src/config_page.html new file mode 100644 index 0000000..17d9d74 --- /dev/null +++ b/crates/meta-feeder-sdk/src/config_page.html @@ -0,0 +1,268 @@ + + + + + +feeder config + + + +

Plugin configuration

+

Secrets are write-only — set values are shown as “set”, never echoed back. Saving requires a feeder restart to take effect. Fields marked * are required.

+ +
+ + + + + diff --git a/crates/meta-feeder-sdk/src/enrich.rs b/crates/meta-feeder-sdk/src/enrich.rs new file mode 100644 index 0000000..94f6119 --- /dev/null +++ b/crates/meta-feeder-sdk/src/enrich.rs @@ -0,0 +1,614 @@ +//! Feeder-driven enrichment — the "smart feeder / dumb gateway" pipeline. +//! +//! A feeder that self-publishes (has a `META_CORE_URL`) writes its own base +//! record, then drives the **reused meta-sort enrichment plugins** +//! (`metamesh-plugin-filename-parser`, `metamesh-plugin-tmdb`) over HTTP. Each +//! plugin is handed the `metaCoreUrl` and writes its own fields straight back to +//! meta-core (`PATCH /meta/{cid}` merge), so the gateway never touches +//! enrichment — full separation of concerns. +//! +//! ## Ordering — why we poll +//! +//! tmdb searches on `originalTitle`/`movieYear`/`videoType` (which it reads from +//! the record), and those are produced by filename-parser. So the pipeline is +//! **filename-parser → tmdb**. The plugin `/process` contract is *async* +//! (returns `accepted`, then writes to meta-core and POSTs a callback), so after +//! kicking filename-parser we **poll meta-core** until `originalTitle` appears +//! (bounded), read the merged fields back, and hand them to tmdb. Polling keeps +//! the feeder stateless — no callback-correlation bookkeeping. The plugins' +//! callback POSTs still fire; they hit the feeder's no-op `/enrich/callback`. +//! +//! The whole pipeline runs in a spawned background task so it never adds latency +//! to the libp2p resolve path; the meta-share ingester propagates the enriched +//! record on its next poll (~eventually consistent). + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use tracing::{debug, warn}; + +use crate::meta_core::{self, Provenance}; +use crate::types::DiscoveryRecord; + +/// Operator-supplied enrichment wiring, parsed once from the feeder env. +#[derive(Clone, Debug)] +pub struct EnrichmentConfig { + /// meta-core root the feeder self-publishes to + hands to the plugins. + pub meta_core_url: String, + /// `metamesh-plugin-filename-parser` base URL (optional). + pub filename_parser_url: Option, + /// `metamesh-plugin-tmdb` base URL (optional). + pub tmdb_url: Option, + /// TMDB v3 key / v4 bearer token, POSTed to tmdb `/configure`. + pub tmdb_token: Option, + /// TMDB metadata language (default `en-US`). + pub tmdb_language: String, + /// Stable per-deployment id stamped into `provenance.gatewayPeer`. + pub gateway_peer: String, + /// URL the plugins POST their completion callback to (the feeder's own + /// no-op `/enrich/callback`). Polling is the real completion signal; this + /// only keeps the plugins from logging a failed-callback warning. + pub callback_url: String, + /// `metamesh-plugin-opensubtitles` base URL (optional). When set, the + /// pipeline drives subtitle fetching after tmdb resolves the ids — the + /// single reference implementation of OpenSubtitles fetching (the gateway + /// no longer carries its own). + pub opensubtitles_url: Option, + /// OpenSubtitles consumer API key, POSTed to the plugin's `/configure`. + pub opensubtitles_api_key: Option, + /// OpenSubtitles account username (required by the plugin for downloads). + pub opensubtitles_username: Option, + /// OpenSubtitles account password. + pub opensubtitles_password: Option, + /// Wanted subtitle languages (CSV of ISO 639-1 codes; default `en`). + pub opensubtitles_languages: String, +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +/// Derive a co-bundled sidecar plugin URL from the feeder's own hostname, by the +/// app-compose convention `-feeder` → `-:8080` +/// (e.g. `metafeedertorznab-feeder` → `http://metafeedertorznab-tmdb:8080`). Lets +/// a feeder reach its bundled filename-parser / tmdb sidecars with **no explicit +/// `*_PLUGIN_URL` env** — config lives in the UI, wiring is derived. Returns +/// `None` when `HOSTNAME` is unset or not in `-feeder` form, in which case +/// the caller falls back to the env seed (then soft-skips that plugin). +fn derive_sidecar_url(plugin: &str) -> Option { + let host = env_nonempty("HOSTNAME")?; + let prefix = host.strip_suffix("-feeder")?; + Some(format!("http://{prefix}-{plugin}:8080")) +} + +impl EnrichmentConfig { + /// Parse from the feeder env. Returns `None` when `META_CORE_URL` is unset — + /// the **bundled-feeder** path (records returned over HTTP, the gateway + /// dispatcher stores them). Self-publishing feeders that source meta-core from + /// their own dashboard config use [`from_meta_core`](Self::from_meta_core). + pub fn from_env() -> Option { + Some(Self::from_meta_core(env_nonempty("META_CORE_URL")?)) + } + + /// Build with an explicit meta-core root — sourced from the feeder's own + /// `config.json` (dashboard) when set, else the `META_CORE_URL` env seed. The + /// sidecar URLs come from their `*_PLUGIN_URL` env, falling back to the + /// hostname-derived co-bundled sidecar (see [`derive_sidecar_url`]); the rest + /// keep their env seeds / defaults. Secrets (tmdb token, language) are + /// overlaid by the caller from the per-feeder config. + pub fn from_meta_core(meta_core_url: String) -> Self { + EnrichmentConfig { + meta_core_url, + filename_parser_url: env_nonempty("FILENAME_PARSER_URL") + .or_else(|| derive_sidecar_url("filename-parser")), + tmdb_url: env_nonempty("TMDB_PLUGIN_URL").or_else(|| derive_sidecar_url("tmdb")), + tmdb_token: env_nonempty("TMDB_TOKEN").or_else(|| env_nonempty("PROWLARR_TMDB_TOKEN")), + tmdb_language: env_nonempty("TMDB_LANGUAGE").unwrap_or_else(|| "en-US".to_string()), + gateway_peer: env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()), + callback_url: env_nonempty("META_FEEDER_CALLBACK_URL") + .unwrap_or_else(|| "http://127.0.0.1:8080/enrich/callback".to_string()), + opensubtitles_url: env_nonempty("OPENSUBTITLES_PLUGIN_URL"), + opensubtitles_api_key: env_nonempty("OPENSUBTITLES_API_KEY"), + opensubtitles_username: env_nonempty("OPENSUBTITLES_USERNAME"), + opensubtitles_password: env_nonempty("OPENSUBTITLES_PASSWORD"), + opensubtitles_languages: env_nonempty("OPENSUBTITLES_LANGUAGES") + .unwrap_or_else(|| "en".to_string()), + } + } +} + +/// How long to wait for filename-parser to merge `originalTitle` before handing +/// off to tmdb. filename-parser is a pure, fast string parse; this is a generous +/// ceiling, not an expected wait. +const FILENAME_PARSE_POLL_TIMEOUT: Duration = Duration::from_secs(5); +/// How long to wait for tmdb to merge `tmdbid` before fanning its anchor to the +/// sibling files. tmdb hits the network, so a longer ceiling than filename. +const TMDB_POLL_TIMEOUT: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Show-level TMDB fields that are **fanned** from the primary file to every +/// sibling file in a multi-file (season-pack) torrent — they're identical for +/// every episode of a show, so we resolve tmdb **once** and copy these across +/// (avoids N TMDB lookups; the tmdb plugin caches by `cid_midhash256`, which +/// gateway BT records don't carry, so without this each file would re-hit TMDB). +/// Per-file fields (`originalTitle`, `season`, `episode`, `fileName`, `sizeByte`, +/// `quality`, …) are deliberately NOT fanned — filename-parser sets those per file. +const ANCHOR_EXACT: &[&str] = &[ + "tmdbid", + "imdbid", + "poster", + "backdrop", + "rating", + "releasedate", + "studio", + "genres", + "tags", +]; +const ANCHOR_PREFIXES: &[&str] = &["plot/", "genres/", "studio/", "tags/"]; + +/// One file (or pack) to enrich: its CID, the filename to parse, and the base +/// fields already written to meta-core (handed to the plugins as `existingMeta`). +#[derive(Clone, Debug)] +pub struct EnrichTarget { + pub cid: String, + pub file_name: String, + pub base: BTreeMap, +} + +/// Drives meta-core writes + the enrichment plugins for one feeder. Cheap to +/// clone (just a reqwest client + an `Arc`), so it can be moved into a +/// spawned per-resolve task. +#[derive(Clone)] +pub struct Enricher { + http: reqwest::Client, + cfg: Arc, +} + +#[derive(Serialize)] +struct ProcessRequest<'a> { + #[serde(rename = "taskId")] + task_id: String, + cid: &'a str, + #[serde(rename = "filePath")] + file_path: &'a str, + #[serde(rename = "callbackUrl")] + callback_url: &'a str, + #[serde(rename = "metaCoreUrl")] + meta_core_url: &'a str, + #[serde(rename = "existingMeta")] + existing_meta: &'a BTreeMap, +} + +impl Enricher { + pub fn new(http: reqwest::Client, cfg: EnrichmentConfig) -> Self { + Enricher { + http, + cfg: Arc::new(cfg), + } + } + + pub fn meta_core_url(&self) -> &str { + &self.cfg.meta_core_url + } + + /// `true` if any enrichment plugin is wired (so there's enrichment to drive + /// beyond the base-record write). + pub fn has_plugins(&self) -> bool { + self.cfg.filename_parser_url.is_some() || self.cfg.tmdb_url.is_some() + } + + /// Write the feeder's base record into meta-core (merge/upsert) under `cid`. + /// Identical shape to the gateway dispatcher's write (provenance + the + /// `cids/` key-set collapse). + pub async fn write_base_record( + &self, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now( + &record.upstream_id, + &record.record_id, + &self.cfg.gateway_peer, + ); + let body = meta_core::build_metadata_body(record, &prov); + meta_core::put_record(&self.http, &self.cfg.meta_core_url, cid, &body).await + } + + /// Run the video enrichment pipeline for one resolved CID: filename-parser + /// (clean title/year/season/episode) → tmdb (tmdbid/poster/overview). Each + /// plugin writes its own fields to meta-core. Best-effort: any plugin + /// failure is logged and the pipeline continues (the base record already + /// landed, so the item is at worst un-enriched, never lost). + pub async fn enrich_video( + &self, + cid: &str, + file_name: &str, + base_fields: BTreeMap, + ) { + let mut meta = base_fields; + + // 1. filename-parser → originalTitle / movieYear / videoType / season / episode. + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + match self.process(&fp_url, "filename-parser", cid, file_name, &meta).await { + Ok(()) => { + // Poll for the merged fields so tmdb gets the clean title. + if let Some(merged) = self + .poll_until_field(cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + { + meta = merged; + } else { + debug!( + target: "meta-feeder::enrich", + cid, "filename-parser produced no originalTitle within timeout; \ + handing tmdb the base fields" + ); + } + } + Err(e) => warn!( + target: "meta-feeder::enrich", + cid, error = %e, "filename-parser /process failed (non-fatal)" + ), + } + } + + // 2. tmdb → tmdbid / poster / overview / genres (requires fileType=video + + // a clean originalTitle, both now present on `meta`). + if let Some(tmdb_url) = self.cfg.tmdb_url.clone() { + // Configure the API key before each call — tmdb holds it in memory + // only, so this is robust to a tmdb-plugin restart. Cheap (local). + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", cid, file_name, &meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "tmdb /process failed (non-fatal)" + ); + } + } + + // 3. opensubtitles → subtitles// (needs tmdbid/imdbid, which + // tmdb writes async). Poll for tmdbid so the plugin has an id to + // search by, then drive it like any other enrichment plugin. + if self.cfg.opensubtitles_url.is_some() { + let merged = self + .poll_until_field(cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or(meta); + self.enrich_subtitles(cid, file_name, &merged).await; + } + } + + /// Enrich a **bundle** from one torrent (metainfo fan-out): N video files + + /// an optional whole-torrent pack record. filename-parser runs on every + /// target; tmdb runs **once** on the primary video file and its show-level + /// anchor (`tmdbid`/poster/genres/…) is fanned to the sibling files. The + /// pack record gets filename-parse only (no tmdb — it's the non-playable + /// "release as a unit" search handle, D1). Best-effort throughout. + pub async fn enrich_bundle(&self, videos: Vec, pack: Option) { + // Pack: filename-parse only, no tmdb. + if let Some(p) = pack { + self.fire_filename(&p).await; + } + + if videos.is_empty() { + return; + } + if videos.len() == 1 { + // Degenerate single-video bundle → the plain pipeline (filename → tmdb). + let v = &videos[0]; + self.enrich_video(&v.cid, &v.file_name, v.base.clone()).await; + return; + } + + // Multi-file: fire filename-parser for every file (fire-and-forget — only + // the primary is polled, the rest write async). + for v in &videos { + self.fire_filename(v).await; + } + + // Primary = largest by `sizeByte` (ties → lowest index, i.e. first seen). + let primary = pick_primary(&videos); + let p = &videos[primary]; + + // Poll the primary for filename-parser's clean title, then tmdb it. + let merged = self + .poll_until_field(&p.cid, "originalTitle", FILENAME_PARSE_POLL_TIMEOUT) + .await + .unwrap_or_else(|| p.base.clone()); + + let Some(tmdb_url) = self.cfg.tmdb_url.clone() else { + return; // no tmdb wired → filename-only enrichment, done. + }; + if let Err(e) = self.configure_tmdb(&tmdb_url).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "tmdb /configure failed"); + } + if let Err(e) = self.process(&tmdb_url, "tmdb", &p.cid, &p.file_name, &merged).await { + warn!(target: "meta-feeder::enrich", cid = %p.cid, error = %e, "primary tmdb /process failed"); + return; + } + + // Read the resolved anchor back off the primary, then fan it to siblings. + let Some(anchor_rec) = self.poll_until_field(&p.cid, "tmdbid", TMDB_POLL_TIMEOUT).await + else { + debug!(target: "meta-feeder::enrich", cid = %p.cid, "primary produced no tmdbid; nothing to fan"); + return; + }; + let anchor = extract_anchor(&anchor_rec); + if anchor.is_empty() { + return; + } + for (i, v) in videos.iter().enumerate() { + if i == primary { + continue; + } + // PUT is merge/upsert — this layers the show-level anchor onto each + // file's record without disturbing its per-file fields. + if let Err(e) = + meta_core::put_record(&self.http, &self.cfg.meta_core_url, &v.cid, &anchor).await + { + warn!(target: "meta-feeder::enrich", cid = %v.cid, error = %e, "anchor fan PUT failed"); + } + } + + // Subtitles per video: each episode needs its own (its season/episode + // differs), but they share the fanned tmdbid. Poll each for tmdbid + // (present on the primary, fanned onto the siblings) and drive the + // opensubtitles plugin. Best-effort + quota-guarded inside the plugin. + if self.cfg.opensubtitles_url.is_some() { + for v in &videos { + let merged = self + .poll_until_field(&v.cid, "tmdbid", TMDB_POLL_TIMEOUT) + .await + .unwrap_or_else(|| v.base.clone()); + self.enrich_subtitles(&v.cid, &v.file_name, &merged).await; + } + } + } + + /// Fire filename-parser `/process` for one target (fire-and-forget; the + /// plugin merges its fields to meta-core async). + async fn fire_filename(&self, t: &EnrichTarget) { + if let Some(fp_url) = self.cfg.filename_parser_url.clone() { + if let Err(e) = self + .process(&fp_url, "filename-parser", &t.cid, &t.file_name, &t.base) + .await + { + warn!(target: "meta-feeder::enrich", cid = %t.cid, error = %e, "filename-parser /process failed"); + } + } + } + + /// POST `/process` to a plugin. Returns once the plugin *accepts* the task + /// (the actual meta-core write is async on the plugin side; the caller polls + /// meta-core for the result when it needs it downstream). + async fn process( + &self, + plugin_url: &str, + plugin: &str, + cid: &str, + file_path: &str, + existing_meta: &BTreeMap, + ) -> anyhow::Result<()> { + let url = format!("{}/process", plugin_url.trim_end_matches('/')); + let body = ProcessRequest { + task_id: format!("{plugin}-{cid}-{}", now_nanos()), + cid, + file_path, + callback_url: &self.cfg.callback_url, + meta_core_url: &self.cfg.meta_core_url, + existing_meta, + }; + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("{plugin} POST {url} returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the tmdb plugin with the API key + language. + async fn configure_tmdb(&self, tmdb_url: &str) -> anyhow::Result<()> { + let Some(token) = self.cfg.tmdb_token.clone() else { + anyhow::bail!("no TMDB token configured"); + }; + let url = format!("{}/configure", tmdb_url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { "apiKey": token, "language": self.cfg.tmdb_language } + }); + let resp = self.http.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("tmdb /configure returned {status}: {text}"); + } + Ok(()) + } + + /// POST `/configure` to the opensubtitles plugin with the API key, account + /// credentials, and wanted languages. + async fn configure_opensubtitles(&self, url: &str) -> anyhow::Result<()> { + let Some(key) = self.cfg.opensubtitles_api_key.clone() else { + anyhow::bail!("no OpenSubtitles API key configured"); + }; + let cfg_url = format!("{}/configure", url.trim_end_matches('/')); + let body = serde_json::json!({ + "config": { + "apiKey": key, + "username": self.cfg.opensubtitles_username, + "password": self.cfg.opensubtitles_password, + "languages": self.cfg.opensubtitles_languages, + } + }); + let resp = self.http.post(&cfg_url).json(&body).send().await?; + if !resp.status().is_success() { + let status = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("opensubtitles /configure returned {status}: {text}"); + } + Ok(()) + } + + /// Drive the opensubtitles plugin for one video CID. Needs a `tmdbid` or + /// `imdbid` in `meta` to search by (the plugin reads them from `existingMeta` + /// and writes the `subtitles//` key-set straight to meta-core). + /// Best-effort: any failure is logged and swallowed. + async fn enrich_subtitles(&self, cid: &str, file_name: &str, meta: &BTreeMap) { + let Some(url) = self.cfg.opensubtitles_url.clone() else { + return; + }; + if !meta.contains_key("tmdbid") && !meta.contains_key("imdbid") { + debug!( + target: "meta-feeder::enrich", + cid, "no tmdbid/imdbid resolved; skipping opensubtitles" + ); + return; + } + if let Err(e) = self.configure_opensubtitles(&url).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /configure failed (continuing; may soft-skip)" + ); + } + if let Err(e) = self.process(&url, "opensubtitles", cid, file_name, meta).await { + warn!( + target: "meta-feeder::enrich", + cid, error = %e, "opensubtitles /process failed (non-fatal)" + ); + } + } + + /// Poll `GET /meta/{cid}` until `field` is present (non-empty) or `timeout` + /// elapses; returns the merged field map on success. + async fn poll_until_field( + &self, + cid: &str, + field: &str, + timeout: Duration, + ) -> Option> { + let deadline = SystemTime::now() + timeout; + loop { + match meta_core::get_record(&self.http, &self.cfg.meta_core_url, cid).await { + Ok(Some(fields)) if fields.get(field).is_some_and(|v| !v.is_empty()) => { + return Some(fields); + } + Ok(_) => {} + Err(e) => debug!( + target: "meta-feeder::enrich", + cid, error = %e, "poll get_record failed" + ), + } + if SystemTime::now() >= deadline { + return None; + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } +} + +fn now_nanos() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) +} + +/// Pick the primary video file index: largest by `sizeByte`, ties broken toward +/// the first (lowest index). Falls back to 0 for an all-unsized bundle. +fn pick_primary(videos: &[EnrichTarget]) -> usize { + videos + .iter() + .enumerate() + .max_by_key(|(i, v)| { + let size = v.base.get("sizeByte").and_then(|s| s.parse::().ok()).unwrap_or(0); + // larger size wins; on a tie the smaller index wins (Reverse). + (size, std::cmp::Reverse(*i)) + }) + .map(|(i, _)| i) + .unwrap_or(0) +} + +/// Project a resolved record down to the show-level anchor fields that are safe +/// to copy to sibling files (see [`ANCHOR_EXACT`] / [`ANCHOR_PREFIXES`]). +fn extract_anchor(fields: &BTreeMap) -> BTreeMap { + fields + .iter() + .filter(|(k, _)| { + ANCHOR_EXACT.contains(&k.as_str()) + || ANCHOR_PREFIXES.iter().any(|p| k.starts_with(p)) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_env_requires_meta_core_url() { + // Not set in the test env → None (legacy gateway-stores path). + // (We don't mutate process env here to avoid cross-test races; this just + // documents the contract.) + assert!(EnrichmentConfig::from_env().is_none() || EnrichmentConfig::from_env().is_some()); + } + + fn tgt(cid: &str, size: u64) -> EnrichTarget { + EnrichTarget { + cid: cid.into(), + file_name: cid.into(), + base: BTreeMap::from_iter([("sizeByte".to_string(), size.to_string())]), + } + } + + #[test] + fn pick_primary_is_largest_then_first() { + let v = vec![tgt("a", 100), tgt("b", 900), tgt("c", 900), tgt("d", 50)]; + // 'b' and 'c' tie at 900 → lowest index ('b', idx 1) wins. + assert_eq!(pick_primary(&v), 1); + } + + #[test] + fn pick_primary_all_unsized_defaults_first() { + let v = vec![ + EnrichTarget { cid: "a".into(), file_name: "a".into(), base: BTreeMap::new() }, + EnrichTarget { cid: "b".into(), file_name: "b".into(), base: BTreeMap::new() }, + ]; + assert_eq!(pick_primary(&v), 0); + } + + #[test] + fn extract_anchor_keeps_show_level_drops_per_file() { + let rec = BTreeMap::from_iter([ + ("tmdbid".to_string(), "10378".to_string()), + ("imdbid".to_string(), "tt1254207".to_string()), + ("poster".to_string(), "bafkPOSTER".to_string()), + ("genres".to_string(), "Animation|Comedy".to_string()), + ("plot/eng".to_string(), "a plot".to_string()), + ("tags".to_string(), "tmdb-verified".to_string()), + // per-file — must NOT be fanned: + ("originalTitle".to_string(), "Big Buck Bunny".to_string()), + ("season".to_string(), "1".to_string()), + ("episode".to_string(), "3".to_string()), + ("fileName".to_string(), "ep3.mkv".to_string()), + ]); + let a = extract_anchor(&rec); + assert!(a.contains_key("tmdbid") && a.contains_key("poster") && a.contains_key("plot/eng")); + assert!(a.contains_key("genres") && a.contains_key("tags") && a.contains_key("imdbid")); + assert!(!a.contains_key("originalTitle"), "originalTitle is per-file"); + assert!(!a.contains_key("season") && !a.contains_key("episode")); + assert!(!a.contains_key("fileName")); + } +} diff --git a/crates/meta-feeder-sdk/src/filename_meta.rs b/crates/meta-feeder-sdk/src/filename_meta.rs new file mode 100644 index 0000000..8c93c11 --- /dev/null +++ b/crates/meta-feeder-sdk/src/filename_meta.rs @@ -0,0 +1,1309 @@ +//! Season / episode extraction from a torrent release title. +//! +//! This is a Rust port of `@metazla/filename-tools` +//! (`packages/filename-tool/src/lib/...`) — specifically the +//! `FileNameVideoMetaExtractor.computeSeasonAndEpisodeFromFilename` → +//! `SerieFilesNameAnalysisTool.findSeasonAndEpisode` pipeline plus the +//! `FileNameCleaner` pre-clean steps it depends on. filename-tools is the +//! best season/episode heuristic we have (the meta-sort `torrent` plugin +//! already uses it), so rather than invent a second, weaker heuristic for +//! the gateway we mirror its pattern set and ordering verbatim. +//! +//! ## Differences from the TS original (deliberate, documented) +//! +//! 1. **No parent-folder fallback.** The TS extractor also reads +//! `Season N` out of the *parent directory* name. A torznab hit is a +//! single release title with no directory context, so that branch is +//! dropped — `computeSeasonAndEpisodeFromFilePath`'s folder lookup +//! simply has nothing to read. +//! 2. **Rightmost-number fallback uses the captured digits, not the raw +//! regex match.** The TS `extractRightmostNumber` does +//! `parseFloat(fullMatch)` where the match can include a leading `.` +//! (the `soloEp` boundary fragment is `(?:\.|\b)`), so `parseFloat` +//! can yield e.g. `0.117` for `.117.`. We read capture group 1 (the +//! bare number) instead, which is what the function *means* to return. +//! +//! The caller (`torznab::into_discovery_record`) gates this on TV-ness so +//! a movie's release year can't be misread as an episode number by the +//! rightmost-number fallback. + +use std::collections::BTreeSet; +use std::sync::LazyLock; + +use regex::Regex; + +/// Structured season/episode extracted from a release title. Each field +/// is the stringified number exactly as the TS extractor would emit it +/// (`"" + n`): whole numbers render without a decimal, half-episode +/// specials keep the `.5` (e.g. `"13"`, `"13.5"`). +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SeasonEpisode { + pub season: Option, + pub episode: Option, + /// First episode of a multi-episode **pack/batch** (`(01-10)` → `"1"`). + /// Set together with [`episode_end`]; mutually exclusive with the + /// single-episode [`episode`] field. + pub episode_start: Option, + /// Last episode of a multi-episode pack/batch (`(01-10)` → `"10"`). + pub episode_end: Option, + /// `season * 10000 + episode` disambiguator (matches the TS + /// `increment` field). Computed for completeness; the torznab caller + /// currently only persists `season`/`episode`. + pub increment: Option, + /// True when this release is a **pack/batch** rather than a single + /// episode: an explicit episode range (`(01-10)`, `第29-38話`), a season + /// range (`S01-S03`), or a whole-season release (`Season 3`, `S03` with + /// no episode). Drives `contentKind=pack` (see the torznab caller); a pack + /// is deliberately **not** given a single `episode`. + pub is_pack: bool, + /// True when [`season`] came from an explicit season-bearing token + /// (`S01E05`, `Season 2`, `S03`). An episode with no such token is left + /// season-less (no "assume season 1" default), so a per-file reparse uses + /// this to know it should borrow the season from the pack/parent title (see + /// [`extract_season_episode_with_context`]). + pub season_explicit: bool, +} + +impl SeasonEpisode { + /// A pack carrying a concrete inclusive episode range. + pub fn has_episode_range(&self) -> bool { + self.episode_start.is_some() && self.episode_end.is_some() + } +} + +// --------------------------------------------------------------------------- +// Pattern fragments — copied 1:1 from +// `filename-tool/src/lib/config/SeasonAndEpisodePatterns.ts`. +// --------------------------------------------------------------------------- + +const SW: &str = r"(?:\.|\b)"; // a dot or word boundary +const S: &str = r"(?:\.|\b|\s)"; +const END: &str = r"(?:(?:\.|\b).*)"; // trailing remainder, matched so it's anchored +const EP4: &str = r"((?:\d{1,4})(?:\.5)?)"; // 1..9999(.5) +const EP3: &str = r"((?:\d{1,3})(?:\.5)?)"; // 1..999(.5) +const EP2: &str = r"((?:\d{1,2})(?:\.5)?)"; // 1..99(.5) +const SEA: &str = r"((?:\d{1,2})(?:\.5)?)"; // season 1..99 +const EP4MY: &str = r"((?:\d{1,3}|1[0-7]\d{2}|1800)(?:\.5)?)"; // 1..1800(.5) + +/// Season-AND-episode patterns, longest/most-specific first. Each has +/// exactly two capture groups: group 1 = season, group 2 = episode. +/// Order matters — the first to match wins (mirrors the TS array order). +static SEASON_EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{S}*Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}\[{SEA}{S}*x{S}*{EP4}\]{END}"), + format!(r"(?i){SW}\({SEA}{S}*x{S}*{EP4}\){END}"), + format!(r"(?i){SW}{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*x{S}*{EP4}{END}"), + format!(r"(?i){SW}S{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}{SEA}{S}*E{EP4}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}*E{S}*{EP2}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{S}+{EP2}{END}"), + format!(r"(?i){SW}S{SEA}{S}*-{S}*{EP2}{END}"), + format!(r"(?i){SW}{SEA}{S}*-{S}*{EP2}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season+episode pattern compiles")) + .collect() +}); + +/// Episode-only patterns. Each has exactly one capture group: the episode. +static EPISODE_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Episode{S}*{EP4}{END}"), + format!(r"(?i){SW}Epizoda{S}*{EP4}{END}"), + format!(r"(?i){SW}E{S}*{EP4MY}{END}"), + format!(r"(?i){SW}Ep{EP4MY}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("episode-only pattern compiles")) + .collect() +}); + +/// `soloEp` — the rightmost-number fallback. Capture group 1 is the number. +static SOLO_EP: LazyLock = + LazyLock::new(|| Regex::new(&format!(r"{SW}{EP3}{SW}")).expect("soloEp compiles")); + +// --------------------------------------------------------------------------- +// Episode-range / pack detection. Anime + TV batches encode a *range* of +// episodes (`(01-10)`, `01-28`, `第29-38話`, `26~37`) or a whole season +// (`Season 3`, `S03`). The TS `findSeasonAndEpisode` had no range concept, so +// its `{SEA}-{EP2}` season+episode pattern mis-reads `01-10` as "S1 E10" and a +// batch collapses to one phantom episode. We detect ranges FIRST (before +// `find_season_and_episode`) and emit `episode_start`/`episode_end` + +// `is_pack`, so the caller can stamp `contentKind=pack` instead. +// +// Disambiguation from a real `S1E2`/`1x05`/`S2 - 07`: +// - ranges use a `-`/`~` separator with TWO numbers and `end > start`; +// - the bare form requires the start digit to be preceded by a space/`_`/`-` +// (or string start), so the `2` in `S2 - 07` (preceded by `S`) is NOT read +// as a range start — that release falls through to the single-episode path; +// - 4-digit year-like / out-of-range operands are rejected. +// --------------------------------------------------------------------------- + +/// Bracketed/parenthesised episode range: `(01-10)`, `[01-12]`, `(1~12)`. +static RANGE_BRACKET: LazyLock = LazyLock::new(|| { + Regex::new(r"[\[(]\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*[\])]").expect("bracket range compiles") +}); + +/// CJK episode range: `第29-38話` / `第29~38話`. +static RANGE_CJK: LazyLock = LazyLock::new(|| { + Regex::new(r"第\s*0*(\d{1,4})\s*[-~]\s*0*(\d{1,4})\s*話").expect("cjk range compiles") +}); + +/// Bare episode range with a **tight** separator: `01-28`, `26~37` (no spaces +/// around the `-`/`~`). The start digit must follow `^`/space/`_` so a +/// season-prefixed `S2-07` (the `2` follows `S`) doesn't match. The tight +/// separator is the disambiguator from the very common `Show 2 - 11` / +/// `Show - 11` episode-separator convention (spaced dash → single episode, not +/// a range); spaced ranges only come through the unambiguous bracketed form. +static RANGE_BARE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:^|[\s_])0*(\d{1,4})[-~]0*(\d{1,4})(?:[\s_)\]]|$)") + .expect("bare range compiles") +}); + +/// Season range (`S01-S03`, `S1~S3`) — a whole-series pack with no single +/// season. Matched only as a fallback when no episode range is present. +static SEASON_RANGE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\bS\s*0*(\d{1,2})\s*[-~]\s*S\s*0*(\d{1,2})\b").expect("season range compiles") +}); + +/// `(start, end)` is a plausible inclusive episode range: ascending, and +/// neither operand is a 4-digit year / resolution-like number. +fn plausible_range(start: u32, end: u32) -> bool { + end > start && (1..1900).contains(&start) && end < 1900 +} + +/// Detect an episode range anywhere in the (precleaned) title. Returns the +/// first plausible `(start, end)` found, trying the unambiguous bracketed and +/// CJK forms before the bare form. +fn detect_episode_range(s: &str) -> Option<(u32, u32)> { + for re in [&*RANGE_BRACKET, &*RANGE_CJK, &*RANGE_BARE] { + for caps in re.captures_iter(s) { + let start = caps.get(1).and_then(|m| m.as_str().parse::().ok()); + let end = caps.get(2).and_then(|m| m.as_str().parse::().ok()); + if let (Some(a), Some(b)) = (start, end) { + if plausible_range(a, b) { + return Some((a, b)); + } + } + } + } + None +} + +/// 8-char CRC32 (SFV) marker, e.g. the `[FC412C51]` Nyaa suffix. +static SFV: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b[A-F0-9]{8}\b").expect("sfv compiles")); + +// --------------------------------------------------------------------------- +// Keyword pre-clean — copied from +// `filename-tool/src/lib/config/KeywordsArray.ts`. +// Removed before season/episode detection so release tags like `x264`, +// `1080p`, `720` can't be mistaken for episode numbers by the fallback. +// --------------------------------------------------------------------------- + +/// Word-bounded keywords (resolution/codec/source/group tags). Order is +/// irrelevant once compiled individually, but we sort longest-first to +/// match the TS constructor (`keyword.length` desc) for identical +/// overlap behaviour. +const KEYWORDS: &[&str] = &[ + "VF", + "aXXo", + "STFR", + "SUBFORCED", + "BrRipx264", + "YIFY", + "TRUEFRENCH", + "VFF", + "ATVP", + "RenewalEX", + "BD", + "DD", + "8bits", + "8 bits", + "10bits", + "10 bits", + "part 1", + "part 2", + "part 3", + "part 4", + "part A", + "part B", + "part C", + "part D", + "AAC5", + "AAC5.1", + "1080i", + "1080p", + "2160p", + "480i", + "480p", + "4k", + "576i", + "576p", + "720", + "720i", + "720p", + "aac", + "aac4", + "ac3", + "amzn", + "apple tv+", + "avi", + "bbc", + "bd5", + "bdrip", + "blu-ray", + "bluray", + "brrip", + "cam", + "cw", + "dc", + "dcu", + "ddp5.1", + "director's cut", + "disney+", + "divx", + "divx5", + "dl", + "dsr", + "dsrip", + "dts", + "dual audio", + "dubbed", + "dvd", + "dvdivx", + "dvdr", + "dvdrip", + "dvdscr", + "dvdscreener", + "eng", + "eng sub", + "esp", + "fan edit", + "fhd", + "flv", + "fs", + "ger", + "german", + "h.264", + "h.265", + "h264", + "h265", + "hardcoded", + "hbo", + "hd", + "hddvd", + "hdr", + "hdrip", + "hdtv", + "hdtvrip", + "hevc", + "hq", + "hrhd", + "hrhdtv", + "hulu", + "imax", + "ita", + "jpn", + "ld", + "md", + "mkv", + "mp3", + "mp4", + "mpeg", + "mpg", + "mq", + "multi", + "multisubs", + "netflix", + "nf", + "nfofix", + "ntsc", + "ogg", + "ogm", + "ova", + "pal", + "pdtv", + "r3", + "r5", + "rerip", + "rsvcd", + "screener", + "sd", + "se", + "subbed", + "svcd", + "tc", + "telecine", + "telesync", + "ts", + "tv series", + "uhd", + "uhdtv", + "uhdv", + "v2", + "vcd", + "vostfr", + "web", + "web-dl", + "webcast", + "webrip", + "wmv", + "ws", + "www", + "x264", + "x265", + "xsvcd", + "xvid", + "xvidvd", + "xxx", + "bits", + "AnimeServ", +]; + +/// Substrings removed without word boundaries (the TS `substringArray`). +const SUBSTRINGS: &[&str] = &["v1", "v2", "v3", "v4", "v5"]; + +/// Compiled keyword regexes (`\bKEYWORD\b`, case-insensitive), longest-first. +static KEYWORD_RES: LazyLock> = LazyLock::new(|| { + let mut kws: Vec<&str> = KEYWORDS.to_vec(); + kws.sort_by_key(|k| std::cmp::Reverse(k.len())); + kws.into_iter() + .map(|k| Regex::new(&format!(r"(?i)\b{}\b", regex::escape(k))).expect("keyword compiles")) + .collect() +}); + +/// Compiled substring regexes (no boundaries, case-insensitive), longest-first. +static SUBSTRING_RES: LazyLock> = LazyLock::new(|| { + let mut subs: Vec<&str> = SUBSTRINGS.to_vec(); + subs.sort_by_key(|s| std::cmp::Reverse(s.len())); + subs.into_iter() + .map(|s| Regex::new(&format!(r"(?i){}", regex::escape(s))).expect("substring compiles")) + .collect() +}); + +// --------------------------------------------------------------------------- +// Language tags — anime/torznab upstreams encode language as release-name +// tags (`VOSTFR`, `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a +// structured field. `extract_languages` reads *every* tag off the raw title +// and returns the full set, so a `MULTi VOSTFR` release yields `{mul, fre}`. +// The gateway writes each as a `languages/ = "true"` key-set member +// (METADATA_KEYS §9) that the preferred-language filter matches against. +// +// Patterns are ported from the battle-tested torrent-name parsers +// (dreulavelle/PTT, Radarr `LanguageParser.cs`); the lookbehind/lookahead +// guards those use to dodge false positives (`DTS ES`, `WEB-DL`, `Shang-Chi`, +// `Tel Aviv`, `www.x.tld`) are reproduced here in code, since Rust's `regex` +// crate has no lookaround. Two defensive choices do most of the work: +// 1. **No bare 2-letter codes** (`en`/`de`/`es`/`it`/`fi`/`nl`/`pl`/…). They +// collide with ordinary words, names, and domain TLDs. We rely on the +// 3-letter scene codes (`eng`/`ger`/`ita`) and full names instead. +// 2. **Word boundaries on every tag**, so `\bita\b` can't fire inside +// `digital`, `\bpor\b` inside `portal`, `\bchi\b` inside `Chicago`. +// --------------------------------------------------------------------------- + +/// Website/domain noise (`www.foo.bar/baz`) stripped before matching so a TLD +/// or path token can't be read as a language. Only `www.`-prefixed runs are +/// removed — a generic `host.tld` matcher would eat dotted release names like +/// `Spy.x.Family`. Runs on the raw (still-dotted) title. +static URL_STRIP: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bwww\.\S+").expect("url strip re compiles")); + +/// Guard phrases removed from the separator-normalized title before language +/// matching, because each contains a substring that would otherwise +/// false-positive a tag. None of these phrases is itself a language tag, so +/// removal is loss-free for detection — this is how we reproduce the upstream +/// parsers' lookbehind guards without lookaround: +/// - `shang chi` → guards `\bchi\b` (the film "Shang-Chi", Chinese) +/// - `tel aviv` → guards `\btel\b` (Telugu) +/// - `web dl` / `webdl` → guards `\bdl\b` (German dual-language marker) +static LANG_GUARD_STRIP: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:shang\s+chi|tel\s+aviv|web\s+dl|webdl)\b") + .expect("lang guard re compiles") +}); + +/// Release-title language tags → ISO 639-2/B (alpha-3). **The codes match +/// meta-share's UI language vocabulary** (`ui/js/language-prefs.js` +/// `ISO_639_1_TO_639_2`: `fre`/`ger`/`chi`, not the 639-3 `fra`/`deu`/`zho`), +/// because meta-share's "languages I read" preference is the source of truth +/// the meta-watch filter compares against — the enriched value must be in the +/// same space the chips use. (This deliberately diverges from `gutenberg.rs` / +/// METADATA_KEYS §1's 639-3 convention; aligning those is a separate cleanup.) +/// +/// `mul` (ISO 639-2 "Multiple languages") is the honest marker for +/// `MULTi`/`dual-audio` — we can't enumerate which languages a multi release +/// carries, so we don't guess. A `MULTi VOSTFR` release matches *both* `mul` +/// and `fre`: every pattern that matches contributes its code, so the result +/// is a set, not a first-wins pick. +static LANGUAGE_TAGS: LazyLock> = LazyLock::new(|| { + [ + // Multi / dual-audio → "mul". `dl` matches only after "web dl" has + // been stripped (see LANG_GUARD_STRIP). + ( + r"\bmulti(?:lang)?\b|\bmultisubs?\b|\bdual\s?audio\b|\bdual\b|\bdl\b", + "mul", + ), + // French — the richest scene vocabulary (truefrench/vff/vfq/vostfr…). + ( + r"\btruefrench\b|\bvostfr\b|\bsubfrench\b|\bfrench\b|\bvf[fqi2]?\b|\bstfr\b|\bfra\b|\bfre\b", + "fre", + ), + (r"\benglish\b|\beng\s?sub\b|\beng\b", "eng"), + (r"\bjapanese\b|\bjpn\b|\bjap\b", "jpn"), + (r"\bitalian\b|\bita\b", "ita"), + (r"\bgerman\b|\bger\b|\bdeu\b", "ger"), + ( + r"\bspanish\b|\bcastellano\b|\bespa[nñ]ol\b|\blatino\b|\besp\b|\bspa\b", + "spa", + ), + (r"\bportuguese\b|\bdublado\b|\blegendado\b|\bpor\b", "por"), + (r"\bkorean\b|\bkor\b", "kor"), + ( + r"\bchinese\b|\bmandarin\b|\bcantonese\b|\bchs\b|\bcht\b|\bzho\b|\bchi\b", + "chi", + ), + (r"\brussian\b|\brus\b", "rus"), + (r"\bdutch\b|\bflemish\b|\bnld\b|\bdut\b", "dut"), + (r"\bpolish\b|\bpl\s?dub\b|\bdub\s?pl\b|\blektor\b|\bpol\b", "pol"), + (r"\bczech\b|\bcze\b|\bces\b", "cze"), + (r"\bslovak\b|\bslo\b|\bslk\b", "slo"), + (r"\bhungarian\b|\bhundub\b|\bhun\b", "hun"), + ( + r"\bromanian\b|\brosub(?:bed)?\b|\brodubbed\b|\brum\b|\bron\b", + "rum", + ), + (r"\bgreek\b|\bgre\b|\bell\b", "gre"), + (r"\bukrainian\b|\bukr\b", "ukr"), + (r"\bbulgarian\b|\bbgaudio\b|\bbul\b", "bul"), + (r"\bswedish\b|\bswesub\b|\bswe\b", "swe"), + (r"\bnorwegian\b|\bnorsk\b", "nor"), + (r"\bdanish\b|\bdansk\b", "dan"), + (r"\bfinnish\b|\bsuomi\b", "fin"), + (r"\bturkish\b|\btur\b", "tur"), + (r"\barabic\b|\bara\b", "ara"), + (r"\bhindi\b|\bhin\b", "hin"), + (r"\btamil\b|\btam\b", "tam"), + (r"\btelugu\b|\btel\b", "tel"), + (r"\bmalayalam\b|\bmal\b", "mal"), + (r"\bvietnamese\b|\bvie\b", "vie"), + (r"\bthai\b|\btha\b", "tha"), + ] + .iter() + .map(|(p, code)| { + ( + Regex::new(&format!("(?i){p}")).expect("language tag re compiles"), + *code, + ) + }) + .collect() +}); + +/// Characters `cleanSpecialChars` replaces with a space (note: `.` and +/// `-` are deliberately preserved — they carry `S01.E02` / `Show - 07` +/// structure). +const SPECIAL_CHARS: &[char] = &[ + '\'', ',', '_', '@', '#', '$', '%', '^', '&', '*', '+', '=', '<', '>', '?', '|', ':', ';', '"', + '`', '~', +]; + +/// Extract season/episode from a release title. +/// +/// Faithful to `computeSeasonAndEpisodeFromFilename` → +/// `findSeasonAndEpisode` → `computeSeasonAndEpisodeFromFilePath` +/// (minus the parent-folder branch — see the module doc). +pub fn extract_season_episode(title: &str) -> SeasonEpisode { + let cleaned = preclean(title); + // Range detection runs on the *raw* title (with `~` normalized to `-`), + // not the precleaned one: `preclean` replaces `~` with a space (it's a + // "special char"), which would erase a `26~37` range, and it never touches + // the `(`/`[` brackets the bracketed form anchors on. + let range_src = title.replace('~', "-"); + + // Pack detection runs first — an episode range (`(01-10)`) would otherwise + // be mis-read as `S1E10` by `find_season_and_episode`'s `{SEA}-{EP2}` + // pattern. A pack is never given a single `episode`. + if let Some((start, end)) = detect_episode_range(&range_src) { + let season = find_pack_season(&cleaned); + return SeasonEpisode { + is_pack: true, + episode_start: Some(start.to_string()), + episode_end: Some(end.to_string()), + // An explicit season token (`S2 (01-10)`) buckets the pack; a bare + // range (`Frieren (01-28)`) leaves the season for the caller. + season: season.map(|s| s.to_string()), + season_explicit: season.is_some(), + ..Default::default() + }; + } + // Season range (`S01-S03`) — a whole-series pack, no single season/episode. + if SEASON_RANGE.is_match(&range_src) { + return SeasonEpisode { + is_pack: true, + ..Default::default() + }; + } + + let (sea, ep) = find_season_and_episode(&cleaned); + + let mut out = SeasonEpisode::default(); + // season/episode 0 are valid; only < 0 means "not found". + if sea >= 0.0 { + out.season = Some(fmt_num(sea)); + out.season_explicit = true; + } + if ep >= 0.0 { + out.episode = Some(fmt_num(ep)); + // An episode with no season is left season-LESS on purpose: a season is + // only asserted when explicitly hinted — an S-token in the title here, + // or (via extract_season_episode_with_context) the parent/pack title. + // Absolute-numbered releases ("Show - 28") thus carry an episode and no + // season, so the UI buckets them under "Episodes" rather than a + // fabricated "Season 1". `increment` encodes a season, so it is only + // emitted when the season is known. + if sea >= 0.0 { + out.increment = Some(fmt_num(sea * 10000.0 + ep)); + } + } else if sea >= 0.0 { + // Season present, no episode → a whole-season pack (`Season 3`, `S03`). + out.is_pack = true; + } + out +} + +/// Extract season/episode using a `/` style context — the +/// per-file path of an opened multi-file (pack) torrent. Parses the file's own +/// `filename` first; when the filename yields an episode but **no** season, the +/// season is recovered from the `parent` (the pack/torrent title). This +/// restores the parent-folder season fallback the TS +/// `computeSeasonAndEpisodeFromFilePath` had and this port originally dropped +/// (see the module doc) — so `Frieren S2 [Batch] / Frieren - 03.mkv` resolves +/// to S2E3 rather than the inherited pack-level number. +pub fn extract_season_episode_with_context(parent: &str, filename: &str) -> SeasonEpisode { + let mut file_se = extract_season_episode(filename); + // Only borrow the parent's season for a concrete single episode whose own + // filename carries no *explicit* season token (so it is otherwise + // season-less). A per-file record is an episode, not a pack, so we don't + // propagate the parent's pack/range shape here. + if file_se.episode.is_some() && !file_se.season_explicit { + if let Some(s) = find_pack_season(&preclean(parent)) { + file_se.season = Some(s.to_string()); + file_se.season_explicit = true; + if let Some(ep) = file_se.episode.as_deref().and_then(|e| e.parse::().ok()) { + file_se.increment = Some(fmt_num(s as f64 * 10000.0 + ep)); + } + } + } + file_se +} + +/// Season number parsed from a keyword'd season token (`Season N`, `Saison N`, +/// `S NN`) anywhere in the (precleaned) title — used to bucket a pack and to +/// recover a per-file season from the parent. Excludes the bare-number season +/// form so an episode-only title isn't mis-bucketed. +fn find_pack_season(s: &str) -> Option { + let padded = format!(" {s} "); + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let v = parse_group(&caps, 1); + if v >= 0.0 { + return Some(v as i64); + } + } + } + None +} + +/// Best-effort set of languages a release supports (audio **and** subtitles, +/// mixed), inferred from its title tags. +/// +/// Anime/torznab upstreams encode language as release-name tags (`VOSTFR`, +/// `ENG SUB`, `JPN`, `GER`, `MULTi`, …) rather than a structured field, so we +/// read them off the raw title here. Returns the **full set** of ISO 639-2/B +/// alpha-3 codes found (matching meta-share's UI vocabulary — see +/// [`LANGUAGE_TAGS`]); a `MULTi VOSTFR` release yields `{mul, fre}`. The set is +/// empty when the title carries no recognizable tag — the caller turns that +/// into the `languages/und` sentinel. +pub fn extract_languages(title: &str) -> BTreeSet { + // 1. Strip `www.…` domain noise while the dots are still intact. + let stripped = URL_STRIP.replace_all(title, " "); + // 2. Normalize `.`/`_`/`-` separators to spaces so word-boundary tags + // anchor inside dotted/underscored/hyphenated release names — and so + // the guard phrases ("web dl", "shang chi") match regardless of which + // separator the original used. + let norm: String = stripped + .chars() + .map(|c| if matches!(c, '.' | '_' | '-') { ' ' } else { c }) + .collect(); + // 3. Remove guard phrases that would otherwise false-positive a tag. + let guarded = LANG_GUARD_STRIP.replace_all(&norm, " "); + // 4. Collect every matching code (a set — dedupes, deterministic order). + let mut out = BTreeSet::new(); + for (re, code) in LANGUAGE_TAGS.iter() { + if re.is_match(&guarded) { + out.insert((*code).to_string()); + } + } + out +} + +// --------------------------------------------------------------------------- +// Quality — the resolution *tier* of a release, normalized to a single label +// (`quality` field, METADATA_KEYS §3). Torznab ``s carry no structured +// resolution, so — like season/episode and language — the release title is the +// only source. Mirrors the client-side `parseVersion()` heuristic in +// meta-watch's `detail.js` so a gateway-filled `quality` and the UI's +// title-parse fallback agree. Interlaced (`1080i`/`720i`) and `4k` collapse to +// the progressive/`2160p` tier; an explicit `WxH` token is mapped by height. +// --------------------------------------------------------------------------- + +/// `2160p`/`4k`, `1080p`/`1080i`, `720p`/`720i`, `576p`, `480p`, or a `WxH` +/// token (`1920x1080`) → a normalized `"p"` tier. `None` when the title +/// carries no recognizable resolution. +pub fn extract_quality(title: &str) -> Option { + static TIER: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(2160p|4k|1080p|1080i|720p|720i|576p|480p)\b").expect("tier re compiles") + }); + static WXH: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b\d{3,4}\s*x\s*(\d{3,4})\b").expect("wxh re compiles")); + // Normalize `.`/`_` separators so `\b`-anchored tokens match inside + // dot/underscore releases (`BDRip_1920x1080_x264`, `Show.1080p.x264`). + let t = title.replace(['.', '_'], " "); + if let Some(m) = TIER.captures(&t).and_then(|c| c.get(1)) { + let tok = m.as_str().to_lowercase(); + return Some( + match tok.as_str() { + "4k" => "2160p", + "1080i" => "1080p", + "720i" => "720p", + other => other, + } + .to_string(), + ); + } + if let Some(h) = WXH + .captures(&t) + .and_then(|c| c.get(1)) + .and_then(|m| m.as_str().parse::().ok()) + { + let tier = if h >= 2000 { + "2160p" + } else if h >= 1000 { + "1080p" + } else if h >= 700 { + "720p" + } else if h >= 500 { + "576p" + } else { + "480p" + }; + return Some(tier.to_string()); + } + None +} + +/// Extract the video codec from a release title (METADATA_KEYS.md `codec`, a §12 +/// structured search facet). Title-derived / best-effort — same provenance and +/// caveats as [`extract_quality`] (the authoritative per-stream codec lives in +/// `stream/{n}` for locally-ingested files). Normalized to ffmpeg-style tokens +/// so a `codec:hevc` query matches both encoder spellings (`x265`/`H.265`) and +/// any stream-derived value. +pub fn extract_codec(title: &str) -> Option { + static CODEC: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(x265|h\s?265|hevc|x264|h\s?264|avc|av1|xvid|divx|vp9)\b") + .expect("codec re compiles") + }); + // Normalize `.`/`_` separators (and the resulting `H 264`) so `\b`-anchored + // tokens match inside dot/underscore releases (`Show.x265.WEB`, `M_H.264`). + let t = title.replace(['.', '_'], " "); + let tok = CODEC.captures(&t)?.get(1)?.as_str().to_lowercase(); + let cleaned = tok.replace(' ', ""); + let norm = match cleaned.as_str() { + "x265" | "h265" | "hevc" => "hevc", + "x264" | "h264" | "avc" => "h264", + other => other, // av1, xvid, divx, vp9 + }; + Some(norm.to_string()) +} + +/// `removeKeywordsFromFilename` + `removeSFV` + `cleanSpecialChars` + +/// `cleanSpace`, in that order. +fn preclean(filename: &str) -> String { + let mut s = filename.to_string(); + for re in KEYWORD_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + s = re.replace_all(&s, "").into_owned(); + } + // removeSFV — first occurrence only (TS uses a non-global replace). + s = SFV.replace(&s, "").into_owned(); + // cleanSpecialChars — replace each special char with a space. + s = s + .chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect(); + // cleanSpace — collapse runs of whitespace and trim. + s.split_whitespace().collect::>().join(" ") +} + +/// `findSeasonAndEpisode`: returns `(season, episode)` as floats, `-1.0` +/// for "not found". +fn find_season_and_episode(filename: &str) -> (f64, f64) { + // Pad with spaces so word-boundary fragments anchor at the ends + // exactly as the TS code does (`" " + filename + " "`). + let padded = format!(" {filename} "); + + for re in SEASON_EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + let episode = parse_group(&caps, 2); + return (season, episode); + } + } + for re in EPISODE_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let episode = parse_group(&caps, 1); + return (-1.0, episode); + } + } + // Season-only pack (`Season 3`, `Saison 3`, `S03`) with no episode. Caught + // here — *before* the bare rightmost-number fallback — so a whole-season + // pack reports `(season, no-episode)` instead of the fallback mis-reading + // its season digit as an episode and `extract_season_episode` then + // defaulting season to 1 (the `Spy x Family Season 3 (S03)` -> `S01E03` + // bug). The TS pipeline relied on a parent-folder lookup for this case, + // which the gateway dropped (see module doc). + for re in SEASON_WORD_ONLY_PATTERNS.iter() { + if let Some(caps) = re.captures(&padded) { + let season = parse_group(&caps, 1); + if season >= 0.0 { + return (season, -1.0); + } + } + } + // Last hope: the rightmost bare number. `if (number)` in the TS is + // falsy for 0, so a lone 0 doesn't count as an episode. + if let Some(n) = rightmost_number(&padded) { + if n != 0.0 { + return (-1.0, n); + } + } + (-1.0, -1.0) +} + +fn parse_group(caps: ®ex::Captures, idx: usize) -> f64 { + caps.get(idx) + .and_then(|m| m.as_str().parse::().ok()) + .unwrap_or(-1.0) +} + +/// `extractRightmostNumber` — the last `soloEp` match's number. +fn rightmost_number(s: &str) -> Option { + SOLO_EP + .captures_iter(s) + .filter_map(|c| c.get(1)) + .last() + .and_then(|m| m.as_str().parse::().ok()) +} + +/// Stringify like JS `"" + n`: `2.0 -> "2"`, `1.5 -> "1.5"`. Rust's f64 +/// `Display` already drops the trailing `.0`, so this is a thin wrapper +/// kept for intent. +fn fmt_num(n: f64) -> String { + format!("{n}") +} + +// --------------------------------------------------------------------------- +// Title cleaning — port of `FileNameVideoMetaExtractor.cleanTitle` + +// `SerieFilesNameAnalysisTool.{removeBeyondHyphen, removeSeasonAndEpisode}`. +// The gateway previously used a separate hand-rolled `clean_torrent_title` +// that missed underscores/dots, the keyword list, `removeBeyondHyphen`, and +// season-word/episode-range stripping — so TMDB enrichment failed on many +// release-name shapes. Reuse the already-ported building blocks instead. +// --------------------------------------------------------------------------- + +/// Season-only patterns (TS `seasonPatterns`). Each removes a `Season N` / +/// `S N` / trailing bare number and everything after it. +static SEASON_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + format!(r"(?i){SW}{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-only pattern compiles")) + .collect() +}); + +/// Season-only patterns restricted to the **keyword'd** forms (`Season N`, +/// `Saison N`, `S NN`) — used by [`find_season_and_episode`] to recognise a +/// whole-season *pack* (no episode) before the bare rightmost-number fallback +/// fires. Deliberately excludes [`SEASON_ONLY_PATTERNS`]'s 4th, bare-`{SEA}` +/// entry: that one matches any lone number and would turn an episode-only +/// title (`Show - 37`) into a bogus season. `SxxEyy` never reaches here because +/// [`SEASON_EPISODE_PATTERNS`] is tried first in `find_season_and_episode`. +static SEASON_WORD_ONLY_PATTERNS: LazyLock> = LazyLock::new(|| { + [ + format!(r"(?i){SW}Season{S}*{SEA}{END}"), + format!(r"(?i){SW}Saison{S}*{SEA}{END}"), + format!(r"(?i){SW}S{S}*{SEA}{END}"), + ] + .iter() + .map(|p| Regex::new(p).expect("season-word-only pattern compiles")) + .collect() +}); + +/// `cleanTags` — bracketed/parenthesised/braced groups. +static TAG_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\[[^\]]*\]|\([^)]*\)|\{[^}]*\}").expect("tag re compiles")); +/// `removeTagSeparator` — stray bracket chars. +static TAG_SEP_RE: LazyLock = + LazyLock::new(|| Regex::new(r"[\[\]\(\)\{\}]").expect("tag-sep re compiles")); +/// `removeYear` — a standalone 19xx/20xx. +static YEAR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:19|20)\d{2}\b").expect("year re compiles")); +/// `removeBeyondHyphen` — capture everything before the first ` - ` +/// (space-delimited) hyphen. +static BEYOND_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^(.*?)(?:\s+-\s+|\s+-)").expect("beyond-hyphen re compiles")); +static TRAILING_HYPHEN_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\s*-\s*$").expect("trailing-hyphen re compiles")); + +fn clean_special_chars(s: &str) -> String { + s.chars() + .map(|c| if SPECIAL_CHARS.contains(&c) { ' ' } else { c }) + .collect() +} + +fn remove_keywords(s: &str) -> String { + let mut out = s.to_string(); + for re in KEYWORD_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + for re in SUBSTRING_RES.iter() { + out = re.replace_all(&out, "").into_owned(); + } + out +} + +fn remove_beyond_hyphen(s: &str) -> String { + if let Some(c) = BEYOND_HYPHEN_RE.captures(s) { + return c + .get(1) + .map(|m| m.as_str().trim().to_string()) + .unwrap_or_default(); + } + TRAILING_HYPHEN_RE.replace(s, "").trim().to_string() +} + +fn remove_season_and_episode(s: &str) -> String { + for re in SEASON_EPISODE_PATTERNS + .iter() + .chain(EPISODE_PATTERNS.iter()) + .chain(SEASON_ONLY_PATTERNS.iter()) + { + if re.is_match(s) { + return re.replace(s, "").trim().to_string(); + } + } + // removeRightmostNumber fallback. + SOLO_EP.replace(s, "").trim().to_string() +} + +/// Produce a clean show/movie title from a raw release name, faithful to +/// filename-tools' `cleanTitle` pipeline. This is what should feed a TMDB +/// search — NOT a hand-rolled cleaner. +pub fn clean_title(raw: &str) -> String { + let mut s = clean_special_chars(raw); // `_ ~ : ; , ' "` … → space (keeps `.` `-`) + s = remove_keywords(&s); // 172 release/quality keywords + v1..v5 + s = TAG_RE.replace_all(&s, "").into_owned(); // cleanTags + s = remove_beyond_hyphen(&s); // drop everything after the first " - " + s = TAG_SEP_RE.replace_all(&s, "").into_owned(); // stray brackets + s = YEAR_RE.replace_all(&s, "").into_owned(); // removeYear + s = SFV.replace(&s, "").into_owned(); // removeSFV + s = remove_season_and_episode(&s); // S/E, season-word, episode-range, rightmost number + s = s.replace('.', " "); // cleanDot + let s = s.trim().trim_start_matches('-').trim_end_matches('-'); + s.split_whitespace().collect::>().join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn se(title: &str) -> (Option, Option) { + let r = extract_season_episode(title); + (r.season, r.episode) + } + + #[test] + fn strict_sxxeyy() { + assert_eq!( + se("Game of Thrones S01E02 1080p WEB-DL x264-GROUP"), + (Some("1".into()), Some("2".into())) + ); + } + + #[test] + fn dotted_release_name() { + assert_eq!( + se("The.Wire.S03E07.720p.BluRay.x264"), + (Some("3".into()), Some("7".into())) + ); + } + + #[test] + fn clean_title_collapses_release_variants() { + let ct = |s: &str| clean_title(s).to_lowercase(); + // Season-word, underscore/range, dotted+EnN, and the trailing-num + // Nyaa shape all collapse to the bare show title (the cases that + // were failing TMDB enrichment). + assert_eq!( + ct("[yolerejiju] Spy x Family Season 1 (S01) (WEBRip 1080p x265 10bit)"), + "spy x family" + ); + assert_eq!( + ct("[Kashikoi]_Spy_X_Family_-_26~37_(S2 01~12)_[BDRip 1080p]"), + "spy x family" + ); + assert_eq!( + ct("Spy.x.Family.E31.El.Temible.Creuer.WEBRip.x264-H3AsO"), + "spy x family" + ); + assert_eq!( + ct("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p].mkv"), + "spy x family" + ); + } + + #[test] + fn concise_nxnn() { + assert_eq!( + se("Some Show 1x05 720p x264"), + (Some("1".into()), Some("5".into())) + ); + } + + #[test] + fn bracketed_concise() { + assert_eq!( + se("Some Show [2x10] HDTV"), + (Some("2".into()), Some("10".into())) + ); + } + + #[test] + fn verbose_season_episode() { + assert_eq!( + se("My Show Season 2 Episode 13"), + (Some("2".into()), Some("13".into())) + ); + } + + #[test] + fn anime_trailing_number_is_season_less() { + // No season marker → episode via rightmost-number fallback, and NO + // season is fabricated (absolute numbering). SFV id stripped, brackets + // ignored. The UI buckets a season-less episode under "Episodes". + assert_eq!( + se("[SubsPlease] Some Anime - 117 [FC412C51]"), + (None, Some("117".into())) + ); + } + + #[test] + fn half_episode_special_is_preserved() { + assert_eq!( + se("Some Anime S01E13.5"), + (Some("1".into()), Some("13.5".into())) + ); + } + + #[test] + fn increment_is_season_times_10000_plus_episode() { + let r = extract_season_episode("Show S02E05"); + assert_eq!(r.increment.as_deref(), Some("20005")); + } + + #[test] + fn four_digit_year_is_not_misread_as_episode() { + // The `soloEp` fallback caps at 3 digits, so a 4-digit release + // year can't be picked up — no episode here. + let r = extract_season_episode("The Matrix 1999 1080p BluRay x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn short_number_in_movie_title_is_grabbed_by_fallback() { + // A bare 1-3 digit number in a movie title DOES trip the + // rightmost-number fallback (here: "13"). This is exactly why the + // torznab caller gates season/episode on TV-ness — a movie record + // must not inherit this bogus episode. + let r = extract_season_episode("Apollo 13 1080p BluRay x264"); + assert_eq!(r.episode.as_deref(), Some("13")); + } + + #[test] + fn keyword_strip_prevents_codec_false_positive() { + // "x264" must be removed so "264" isn't read as the episode. + let r = extract_season_episode("Plain Movie Title x264"); + assert_eq!(r.episode, None); + } + + #[test] + fn no_numbers_yields_nothing() { + let r = extract_season_episode("Just A Plain Title"); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + } + + #[test] + fn season_only_pack_is_not_misread_as_episode() { + // Whole-season packs (no episode) must report the season with NO + // episode — not `S01E0N` from the rightmost-number fallback. This is + // the original bug: `Spy x Family Season 3 (S03)` was parsed as + // season 1, episode 3, dumping the S3 pack into the Season-1 bucket. + assert_eq!( + se("[yolerejiju] Spy x Family Season 3 (S03) (WEB-DL 1080p H.264 DDP)"), + (Some("3".into()), None) + ); + assert_eq!( + se("[yolerejiju] Spy x Family Season 2 (S02) (WEBRip 1080p x265)"), + (Some("2".into()), None) + ); + // Bare `S03` pack (no `Season` word) via the S-prefixed pattern. + assert_eq!( + se("[Trix] SPY FAMILY S03 v2 (Batch) [WEBRip 1080p AV1 Opus]"), + (Some("3".into()), None) + ); + assert_eq!( + se("Spy.x.Family.S03.MULTi.1080p.WEBRiP.AV1-KAF"), + (Some("3".into()), None) + ); + // French season word. + assert_eq!(se("Une Serie Saison 4 1080p"), (Some("4".into()), None)); + } + + #[test] + fn explicit_sxxeyy_still_wins_over_season_only() { + // `SEASON_EPISODE_PATTERNS` runs first, so a real SxxEyy is unaffected + // by the new season-only branch. + assert_eq!( + se("Spy x Family Season 3 S03E13 1080p"), + (Some("3".into()), Some("13".into())) + ); + } + + #[test] + fn episode_range_is_a_pack_not_sxxeyy() { + // The headline bug: `(01-10)` was mis-read as S1 E10 by the + // `{SEA}-{EP2}` pattern. Now it's a pack with an episode range and no + // single `episode`. + let r = extract_season_episode("Frieren (01-28) [Batch] 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("28")); + + // Bare, dash-delimited, leading-zero range. + let r = extract_season_episode("Some Anime 01-12 1080p"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("12")); + + // Tilde range, no leading zero. + let r = extract_season_episode("[Kashikoi] Spy x Family - 26~37 [BDRip]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("26")); + assert_eq!(r.episode_end.as_deref(), Some("37")); + } + + #[test] + fn explicit_season_buckets_a_pack() { + // `S2 (01-10)` → pack in season 2, episodes 1–10. + let r = extract_season_episode("[grp] Frieren S2 (01-10) (WEBRip 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode_start.as_deref(), Some("1")); + assert_eq!(r.episode_end.as_deref(), Some("10")); + } + + #[test] + fn cjk_episode_range_is_a_pack() { + let r = extract_season_episode("[北宇治字幕组] Frieren [第29-38話][合集][1080p]"); + assert!(r.is_pack); + assert_eq!(r.episode_start.as_deref(), Some("29")); + assert_eq!(r.episode_end.as_deref(), Some("38")); + } + + #[test] + fn season_range_is_a_pack_with_no_single_season() { + let r = extract_season_episode("Frieren S01-S03 Complete 1080p"); + assert!(r.is_pack); + assert_eq!(r.season, None); + assert_eq!(r.episode, None); + assert_eq!(r.episode_start, None); + } + + #[test] + fn whole_season_release_is_a_pack() { + // A season-only release (`Season 3`, `S03`, no episode) is a pack too — + // the original "season-level release shows as one media card" bug. + let r = extract_season_episode("[grp] Spy x Family Season 3 (S03) (WEB-DL 1080p)"); + assert!(r.is_pack); + assert_eq!(r.season.as_deref(), Some("3")); + assert_eq!(r.episode, None); + } + + #[test] + fn single_episode_is_not_a_pack() { + // The disambiguation guards: `S01E02`, `1x05`, and `S2 - 07` are all + // single episodes, never packs. + for t in ["Game of Thrones S01E02 1080p", "Some Show 1x05 720p"] { + assert!(!extract_season_episode(t).is_pack, "{t} mis-detected as pack"); + } + let r = extract_season_episode("[Greek-Nakama] Frieren S2 - 07 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("7")); + + // The `Show - ` anime convention (spaced dash) is a + // single episode, NOT a `2..11` range — the tight-separator rule keeps + // this out of pack detection. + let r = extract_season_episode("[Naruto-Kun.Hu] Spy x Family 2 - 11 [1080p]"); + assert!(!r.is_pack); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("11")); + + // A year range in a movie title is not an episode range. + assert!(!extract_season_episode("Some Doc 2010-2011 1080p").is_pack); + } + + #[test] + fn per_file_context_recovers_season_from_parent() { + // Opened pack: the inner file carries the episode, the pack title the + // season. `Frieren - 03.mkv` alone is season-less; with the S2 pack + // parent it resolves to S2E3 (the parent is the "season hint"). + let r = extract_season_episode_with_context("Frieren S2 (01-10) [Batch]", "Frieren - 03.mkv"); + assert_eq!(r.season.as_deref(), Some("2")); + assert_eq!(r.episode.as_deref(), Some("3")); + assert!(!r.is_pack); + + // The file's own explicit season wins over the parent. + let r = extract_season_episode_with_context("Frieren S2 [Batch]", "Frieren S01E05.mkv"); + assert_eq!(r.season.as_deref(), Some("1")); + assert_eq!(r.episode.as_deref(), Some("5")); + } + + #[test] + fn absolute_numbered_anime_is_season_less() { + // No season keyword → the rightmost-number fallback still finds the + // episode, but NO season is fabricated. Season is only ever asserted + // from an explicit token (title or parent), never inferred — so an + // absolute-numbered release lands in the UI's "Episodes" bucket. + assert_eq!( + se("[Naruto-Kun.Hu] Spy x Family - 37 [1080p].mkv"), + (None, Some("37".into())) + ); + } + + fn langs(s: &str) -> BTreeSet { + extract_languages(s) + } + + #[test] + fn language_from_release_tags() { + // ISO 639-2/B codes — match meta-share's UI vocabulary (fre/ger/chi). + assert!(langs("[SubsPlease] Some Anime - 117 VOSTFR [FC412C51]").contains("fre")); + assert!(langs("Some.Movie.GERMAN.1080p.BluRay.x264").contains("ger")); + assert!(langs("Show ENG SUB 720p").contains("eng")); + assert!(langs("Spy.x.Family.JPN.1080p").contains("jpn")); + assert!(langs("Film TRUEFRENCH 1080p").contains("fre")); + } + + #[test] + fn multi_is_mul_and_collects_every_language() { + // `languages` is the union of all supported audio+sub languages. + // MULTi → "mul" (ISO 639-2 multiple); a co-present specific tag is + // ALSO collected (the set, not a first-wins pick). + assert_eq!( + langs("Show MULTI VOSTFR 1080p"), + BTreeSet::from(["mul".to_string(), "fre".to_string()]) + ); + assert!(langs("Spy x Family S01E05 MULTi 1080p").contains("mul")); + assert!(langs("Some Anime - 12 [Dual Audio] 1080p").contains("mul")); + // A bilingual release collects both specific codes. + assert_eq!( + langs("Movie.2021.GERMAN.ENGLISH.1080p"), + BTreeSet::from(["ger".to_string(), "eng".to_string()]) + ); + } + + #[test] + fn no_language_tag_yields_empty_set() { + // A plain quality/codec release name carries no language tag — the + // caller maps the empty set to the `languages/und` sentinel. + assert!(langs("Game of Thrones S01E02 1080p WEB-DL x264-GROUP").is_empty()); + // Short codes embedded in words/names must not false-positive: "ita" + // inside "digital", "por" inside "portal", etc. + assert!(langs("VFX Reel 2024 1080p").is_empty()); + assert!(langs("Digital Capital Portal 1080p").is_empty()); + } + + #[test] + fn lookaround_guards_reproduced_in_code() { + // WEB-DL must not read as `dl`→mul (German dual-language marker). + assert!(!langs("Movie 2021 WEB-DL x264").contains("mul")); + // "Shang-Chi" must not read as `chi` (Chinese)… + assert!(!langs("Shang-Chi.2021.1080p.BluRay").contains("chi")); + // …but a real CHI tag still does. + assert!(langs("Movie.2021.CHI.1080p").contains("chi")); + // "Tel Aviv" must not read as `tel` (Telugu). + assert!(!langs("Tel Aviv On Fire 2019 1080p").contains("tel")); + // www. domain noise (incl. its TLD) must not contribute a language. + assert!(langs("www.Torrenting.it Plain Movie 1080p").is_empty()); + } + + #[test] + fn quality_tiers_and_normalization() { + let q = |s: &str| extract_quality(s); + assert_eq!( + q("Game of Thrones S01E02 1080p WEB-DL x264").as_deref(), + Some("1080p") + ); + assert_eq!( + q("The.Wire.S03E07.720p.BluRay.x264").as_deref(), + Some("720p") + ); + // 4k / interlaced collapse to the canonical progressive tier. + assert_eq!(q("Some Movie 2160p").as_deref(), Some("2160p")); + assert_eq!(q("Some Movie 4K HDR").as_deref(), Some("2160p")); + assert_eq!(q("Old Broadcast 1080i HDTV").as_deref(), Some("1080p")); + // WxH fallback, mapped by height; dot/underscore separators handled. + assert_eq!(q("BDRip_1920x1080_x264").as_deref(), Some("1080p")); + assert_eq!(q("Show 1280x720").as_deref(), Some("720p")); + // No resolution token → None (never guess). + assert_eq!(q("Just A Plain Title"), None); + } + + #[test] + fn codec_normalization() { + let c = |s: &str| extract_codec(s); + // HEVC encoder spellings collapse to the codec name. + assert_eq!(c("Movie.2020.1080p.x265-GROUP").as_deref(), Some("hevc")); + assert_eq!(c("Movie 2020 HEVC 10bit").as_deref(), Some("hevc")); + assert_eq!(c("Movie.2020.H.265.WEB").as_deref(), Some("hevc")); + // AVC encoder spellings collapse to h264. + assert_eq!(c("Show.S01E01.720p.x264").as_deref(), Some("h264")); + assert_eq!(c("Show S01E01 H 264 AAC").as_deref(), Some("h264")); + // Others kept as-is. + assert_eq!(c("Clip 2023 AV1 Opus").as_deref(), Some("av1")); + assert_eq!(c("Old Release XviD").as_deref(), Some("xvid")); + // No codec token → None (never guess). + assert_eq!(c("Just A Plain Title 1080p"), None); + } +} diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs new file mode 100644 index 0000000..f958afb --- /dev/null +++ b/crates/meta-feeder-sdk/src/hash.rs @@ -0,0 +1,574 @@ +//! Content-addressed identifiers used by feeder plugins' `compute_outcomes` +//! to hash upstream-fetched bytes. +//! +//! Two function families live here: +//! +//! - [`compute_midhash256`] / [`compute_midhash256_from_sample`] — +//! the size-prefix-plus-middle-1MB-sample identification primitive shared +//! with `meta-sort` / `meta-share` / meta-core. Custom multicodec `0x1000`. +//! Fast but not IPFS-interop. The `_from_sample` entrypoint takes the +//! middle slice directly so plugins that can only fetch a partial range +//! (BitTorrent BEP 9, sliced HTTP) don't have to buffer the whole file +//! just to throw most of it away. **Rust-only by construction.** +//! - [`compute_ipfs_cid`] is the standard IPFS CIDv1 produced over the full +//! bytes — raw codec `0x55` for a single-chunk file, UnixFS dag-pb codec +//! `0x70` rooted on chunked raw-codec leaves for larger files. Output matches +//! `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +//! --hash=sha2-256 `. +//! +//! Cross-implementation parity for `compute_midhash256` with the TypeScript +//! `FastHash.ts` is load-bearing — keep the fixture tests pinned, a regression +//! silently breaks dedupe across the platform. + +use bytes::Bytes; +use sha2::{Digest, Sha256}; + +/// Sample window in bytes (1 MiB). Files this size or smaller are hashed +/// in full; larger files are hashed over the middle 1 MiB slice. +const SAMPLE_SIZE: usize = 1024 * 1024; + +/// Per-block IPFS chunk size (256 KiB), matches kubo's +/// `--chunker=size-262144` default. +pub const IPFS_CHUNK_SIZE: usize = 256 * 1024; + +/// dag-pb fanout — children per internal node. Matches kubo's balanced +/// builder default. +pub const IPFS_FANOUT: usize = 174; + +/// Output of [`compute_ipfs_blocks`]. Carries the root CID **plus every +/// intermediate block** (leaf chunks AND internal dag-pb nodes) keyed by +/// their own CID. Pass `.blocks` to the gateway's blockstore so peers +/// can fetch the file by CID via bitswap. +#[derive(Debug, Clone)] +pub struct IpfsBlocks { + /// Canonical "bafy…" / "bafk…" root cid string. Identical to what + /// [`compute_ipfs_cid`] returns for the same input. + pub root: String, + /// Every block — root included — keyed by its own cid. For a + /// single-leaf file, this is a one-entry vec `[(root, payload)]` + /// (raw codec, payload is the file bytes). For multi-leaf files, + /// it is `leaves ++ internal_nodes` in build order; the **last** + /// entry is the root. + pub blocks: Vec<(String, Bytes)>, +} + +/// Compute a midhash256 CID matching the TypeScript implementation in +/// `meta-hash/src/lib/file-id/FastHash.ts`: +/// +/// 1. Build the hash input as `[size:u64-be][middle 1 MiB sample]` (or the +/// whole file if it fits in 1 MiB). +/// 2. SHA-256 it. +/// 3. Wrap in CIDv1 with custom multicodec `0x1000` for both the codec field +/// and the multihash hash-code field, length 32, then multibase-base32-lower +/// with the `b` prefix. +/// +/// **Endianness for the size prefix is big-endian** to match TS's +/// `writeBigUInt64BE`. Cross-impl mismatch here would silently mangle every +/// import — keep the test pinned. +pub fn compute_midhash256(bytes: &[u8]) -> String { + let size = bytes.len(); + let sample: &[u8] = if size <= SAMPLE_SIZE { + bytes + } else { + let start = (size - SAMPLE_SIZE) / 2; + &bytes[start..start + SAMPLE_SIZE] + }; + compute_midhash256_from_sample(size as u64, sample) +} + +/// Same midhash256 primitive as [`compute_midhash256`], but takes the +/// sample bytes directly rather than slicing them out of a full file. +/// For upstreams that can only deliver a middle range (BitTorrent BEP 9, +/// sliced HTTP), this avoids forcing the caller to buffer the whole file +/// in memory just to throw most of it away. +/// +/// `total_size` is the **full file size in bytes** — used as the +/// size prefix, NOT `middle_sample.len()`. +pub fn compute_midhash256_from_sample(total_size: u64, middle_sample: &[u8]) -> String { + let size_be = total_size.to_be_bytes(); + let mut hasher = Sha256::new(); + hasher.update(size_be); + hasher.update(middle_sample); + let digest: [u8; 32] = hasher.finalize().into(); + + // Multicodec 0x1000 encoded as unsigned varint = [0x80, 0x20]. + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(38); + wire.push(0x01); + wire.extend_from_slice(&[0x80, 0x20]); + wire.extend_from_slice(&[0x80, 0x20]); + wire.push(0x20); + wire.extend_from_slice(&digest); + + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Wrap a 20-byte BitTorrent v1 info-hash as a multibase CIDv1. +/// +/// BT v1 info-hashes ARE SHA-1 digests of the bencoded info dict — so +/// the natural CID form is `codec=raw (0x55)` + `multihash=sha1 (0x11), +/// len=20, digest=`. +pub fn compute_bt_info_cid(infohash_20: &[u8; 20]) -> String { + // CIDv1 layout: [version=0x01][codec varint][multihash code varint][len][digest] + let mut wire = Vec::with_capacity(4 + 20); + wire.push(0x01); // CIDv1 version + wire.push(0x55); // codec: raw + wire.push(0x11); // multihash: sha1 + wire.push(0x14); // multihash length: 20 + wire.extend_from_slice(infohash_20); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a single file inside a **BitTorrent v1** torrent +/// (`btih-v1-file`). MetaMesh-private, adjacent to midhash256's `0x1000`. +pub const BTIH_V1_FILE_CODEC: u64 = 0x1001; + +/// Custom multicodec for a single file inside a **BitTorrent v2** torrent +/// (`btih-v2-file`). +pub const BTIH_V2_FILE_CODEC: u64 = 0x1002; + +/// Encode a `btih-v1-file` CID — a single file inside a BT v1 torrent, +/// addressed by the 20-byte v1 infohash (SHA-1 of the bencoded `info` +/// dict) plus the zero-based file index in `info.files`. Single-file +/// torrents use index `0`. +pub fn compute_bt_v1_file_cid(infohash_20: &[u8; 20], file_index: u64) -> String { + bt_file_cid(BTIH_V1_FILE_CODEC, infohash_20, file_index) +} + +/// Encode a `btih-v2-file` CID — a single file inside a BT v2 torrent +/// (BEP 52), addressed by the 32-byte v2 infohash plus the file's +/// canonical traversal index in `file tree`. +pub fn compute_bt_v2_file_cid(infohash_32: &[u8; 32], file_index: u64) -> String { + bt_file_cid(BTIH_V2_FILE_CODEC, infohash_32, file_index) +} + +/// Shared encoder for the two torrent-file CID families. `infohash` is 20 +/// bytes for v1 / 32 for v2; `codec` selects the family. +fn bt_file_cid(codec: u64, infohash: &[u8], file_index: u64) -> String { + // digest = infohash ‖ varint(file_index) + let mut digest = Vec::with_capacity(infohash.len() + 2); + digest.extend_from_slice(infohash); + write_pb_varint(file_index, &mut digest); + + // CIDv1: [version][codec varint][multihash code varint][len varint][digest] + let mut wire = Vec::with_capacity(1 + 4 + digest.len()); + wire.push(0x01); + write_pb_varint(codec, &mut wire); + write_pb_varint(codec, &mut wire); + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Custom multicodec for a generic **Newznab release** locator (`nzb-release`). +/// MetaMesh-private, adjacent to the torrent-file codecs (`0x1002` is taken by +/// [`BTIH_V2_FILE_CODEC`]). It identifies an indexer *listing* (host + release +/// id), so it is derivable at search time with **no `.nzb` download**. See +/// `USENET-GATEWAY-STUDY.md` §13. (The v1 content-stable `nzb-posting` `0x1003` +/// variant — `sha256(message-ids)` — was removed; only this one is emitted.) +pub const NZB_RELEASE_CODEC: u64 = 0x1004; + +/// Encode an `nzb-release` CID from a Newznab indexer's host + bare release id +/// (the 32-hex `` id). `sha2-256(host ‖ "\n" ‖ id)` — host-namespaced so +/// the same movie on two indexers (different postings) yields different cids. +/// Deterministic + opaque, same wire house-style as the other locator codecs. +pub fn compute_nzb_release_cid(host: &str, release_id: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(host.as_bytes()); + hasher.update(b"\n"); + hasher.update(release_id.as_bytes()); + let digest: [u8; 32] = hasher.finalize().into(); + + // CIDv1: [version=0x01][codec varint][multihash code varint][len=0x20][digest] + let mut wire = Vec::with_capacity(1 + 4 + 1 + 32); + wire.push(0x01); + write_pb_varint(NZB_RELEASE_CODEC, &mut wire); + write_pb_varint(NZB_RELEASE_CODEC, &mut wire); + wire.push(0x20); + wire.extend_from_slice(&digest); + format!("b{}", base32_lower_no_padding(&wire)) +} + +/// Compute a standard IPFS CIDv1 over `bytes`. Output matches kubo's +/// `ipfs add --cid-version=1 --raw-leaves=true --chunker=size-262144 +/// --hash=sha2-256 `. +pub fn compute_ipfs_cid(bytes: &[u8]) -> String { + compute_ipfs_blocks(bytes).root +} + +/// Same wire-format guarantee as [`compute_ipfs_cid`], but exposes every +/// block produced along the way (leaf chunks + internal dag-pb nodes) +/// so callers can populate a bitswap blockstore in one pass. +pub fn compute_ipfs_blocks(bytes: &[u8]) -> IpfsBlocks { + // Leaves: chunk bytes into raw-codec blocks. Empty input still gets + // one (empty) leaf so the cid is well-defined. + let mut blocks: Vec<(String, Bytes)> = Vec::new(); + let leaves: Vec<(Vec, u64)> = if bytes.is_empty() { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(b"")); + blocks.push((cid_string(&cid_wire), Bytes::new())); + vec![(cid_wire, 0)] + } else { + bytes + .chunks(IPFS_CHUNK_SIZE) + .map(|chunk| { + let cid_wire = ipfs_cid_wire(0x55, &sha2_256(chunk)); + blocks.push((cid_string(&cid_wire), Bytes::copy_from_slice(chunk))); + (cid_wire, chunk.len() as u64) + }) + .collect() + }; + + // Single-leaf file: return the leaf cid as the file cid. + if leaves.len() == 1 { + let root = cid_string(&leaves[0].0); + return IpfsBlocks { root, blocks }; + } + + // Multi-leaf: build a balanced tree of UnixFS dag-pb internal nodes. + let mut level: Vec<(Vec, u64, u64)> = leaves + .into_iter() + .map(|(cid, size)| (cid, size, size)) + .collect(); + + while level.len() > 1 { + let mut next: Vec<(Vec, u64, u64)> = + Vec::with_capacity(level.len().div_ceil(IPFS_FANOUT)); + for batch in level.chunks(IPFS_FANOUT) { + let blocksizes: Vec = batch.iter().map(|(_, sz, _)| *sz).collect(); + let filesize: u64 = blocksizes.iter().sum(); + let unixfs_data = encode_unixfs_file(filesize, &blocksizes); + let node_bytes = encode_dagpb_node(batch, &unixfs_data); + let node_cid = ipfs_cid_wire(0x70, &sha2_256(&node_bytes)); + blocks.push((cid_string(&node_cid), Bytes::from(node_bytes.clone()))); + let own_tsize: u64 = + batch.iter().map(|(_, _, t)| *t).sum::() + node_bytes.len() as u64; + next.push((node_cid, filesize, own_tsize)); + } + level = next; + } + let root = cid_string(&level[0].0); + IpfsBlocks { root, blocks } +} + +fn cid_string(wire: &[u8]) -> String { + format!("b{}", base32_lower_no_padding(wire)) +} + +/// CIDv1 wire-form bytes: `[version=0x01][codec varint][multihash code +/// varint=0x12 sha2-256][len=0x20][digest...]`. +fn ipfs_cid_wire(codec: u8, digest: &[u8; 32]) -> Vec { + debug_assert!( + codec < 0x80, + "codec varint must fit in one byte for this helper" + ); + let mut wire = Vec::with_capacity(36); + wire.push(0x01); // CIDv1 + wire.push(codec); + wire.push(0x12); // multihash code: sha2-256 + wire.push(0x20); // digest length: 32 + wire.extend_from_slice(digest); + wire +} + +fn sha2_256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +/// LEB128 unsigned varint encoding into `out`. +fn write_pb_varint(value: u64, out: &mut Vec) { + let mut v = value; + while v >= 0x80 { + out.push((v as u8) | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +fn write_pb_varint_field(field: u32, value: u64, out: &mut Vec) { + write_pb_varint((field as u64) << 3, out); // tag: (field << 3) | wire-type 0 (varint) + write_pb_varint(value, out); +} + +fn write_pb_bytes_field(field: u32, value: &[u8], out: &mut Vec) { + write_pb_varint(((field as u64) << 3) | 2, out); // wire type 2 = length-delimited + write_pb_varint(value.len() as u64, out); + out.extend_from_slice(value); +} + +/// UnixFS protobuf payload for a File node: `Type=File, filesize, +/// blocksizes[]`. +fn encode_unixfs_file(filesize: u64, blocksizes: &[u64]) -> Vec { + let mut out = Vec::new(); + write_pb_varint_field(1, 2, &mut out); // Type = File + write_pb_varint_field(3, filesize, &mut out); // filesize + for &bs in blocksizes { + write_pb_varint_field(4, bs, &mut out); // blocksizes (repeated) + } + out +} + +/// dag-pb PBNode for an internal UnixFS file node. Canonical wire order +/// pinned by the dag-pb spec: `Links` (tag 2) first, `Data` (tag 1) second. +fn encode_dagpb_node(children: &[(Vec, u64, u64)], data: &[u8]) -> Vec { + let mut out = Vec::new(); + for (cid_wire, _filesize, tsize) in children { + let mut link_bytes = Vec::new(); + write_pb_bytes_field(1, cid_wire, &mut link_bytes); // Hash + write_pb_varint_field(3, *tsize, &mut link_bytes); // Tsize + write_pb_bytes_field(2, &link_bytes, &mut out); // Links (tag 2) first + } + write_pb_bytes_field(1, data, &mut out); // Data (tag 1) last + out +} + +/// RFC 4648 base32 with the lowercase alphabet and no padding. Used for the +/// multibase `b` prefix. +pub(crate) fn base32_lower_no_padding(input: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; + let mut out = String::with_capacity(input.len().div_ceil(5) * 8); + let mut buffer: u64 = 0; + let mut bits: u32 = 0; + for &b in input { + buffer = (buffer << 8) | b as u64; + bits += 8; + while bits >= 5 { + bits -= 5; + let idx = ((buffer >> bits) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + } + if bits > 0 { + let idx = ((buffer << (5 - bits)) & 0x1F) as usize; + out.push(ALPHABET[idx] as char); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn midhash256_matches_ts_fixtures() { + assert_eq!( + compute_midhash256(b"hello world"), + "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4" + ); + assert_eq!( + compute_midhash256(b""), + "bagacbabaecxvk4hvugaqw6xxrsxuxrykmyhq35i6ik5pshkn4wzdfdpa5a67y" + ); + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_midhash256(&mb_zeros), + "bagacbabaeawlkt4hn34sxuieoosrnnag3g3wv7gc6alzwfms2nrph2hochrx4" + ); + } + + #[test] + fn midhash256_from_sample_matches_full_bytes() { + let small: &[u8] = b"hello world"; + assert_eq!( + compute_midhash256_from_sample(small.len() as u64, small), + compute_midhash256(small), + ); + + assert_eq!( + compute_midhash256_from_sample(0, b""), + compute_midhash256(b""), + ); + + let one_mb = vec![0u8; SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(one_mb.len() as u64, &one_mb), + compute_midhash256(&one_mb), + ); + + let two_mb = vec![0u8; 2 * SAMPLE_SIZE]; + let start = (two_mb.len() - SAMPLE_SIZE) / 2; + let middle = &two_mb[start..start + SAMPLE_SIZE]; + assert_eq!( + compute_midhash256_from_sample(two_mb.len() as u64, middle), + compute_midhash256(&two_mb), + ); + } + + #[test] + fn midhash256_from_sample_partial_sample_stable() { + let sample = vec![0u8; SAMPLE_SIZE]; + let total_5gb: u64 = 5 * 1024 * 1024 * 1024; + let total_10gb: u64 = 10 * 1024 * 1024 * 1024; + + let h1 = compute_midhash256_from_sample(total_5gb, &sample); + let h2 = compute_midhash256_from_sample(total_5gb, &sample); + assert_eq!(h1, h2, "same (size, sample) must hash identically"); + + let h3 = compute_midhash256_from_sample(total_10gb, &sample); + assert_ne!( + h1, h3, + "size prefix must contribute to cid — same sample, different size" + ); + } + + #[test] + fn bt_v1_file_cid_wire_layout() { + let infohash: [u8; 20] = [ + 0xf9, 0xc8, 0xa7, 0xb6, 0xe5, 0xd4, 0xc3, 0xb2, 0xa1, 0x90, 0x8f, 0x7e, 0x6d, 0x5c, + 0x4b, 0x3a, 0x29, 0x18, 0x07, 0x06, + ]; + + let mut want = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want.extend_from_slice(&infohash); + want.push(0x04); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 4), + format!("b{}", base32_lower_no_padding(&want)) + ); + + let mut want0 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x15]; + want0.extend_from_slice(&infohash); + want0.push(0x00); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want0)) + ); + + let mut want200 = vec![0x01, 0x81, 0x20, 0x81, 0x20, 0x16]; + want200.extend_from_slice(&infohash); + want200.extend_from_slice(&[0xC8, 0x01]); + assert_eq!( + compute_bt_v1_file_cid(&infohash, 200), + format!("b{}", base32_lower_no_padding(&want200)) + ); + } + + #[test] + fn bt_v2_file_cid_wire_layout() { + let infohash: [u8; 32] = [0xAB; 32]; + let mut want = vec![0x01, 0x82, 0x20, 0x82, 0x20, 0x21]; + want.extend_from_slice(&infohash); + want.push(0x00); + assert_eq!( + compute_bt_v2_file_cid(&infohash, 0), + format!("b{}", base32_lower_no_padding(&want)) + ); + } + + #[test] + fn bt_file_cid_properties() { + let ih1 = [0x11u8; 20]; + let ih2 = [0x22u8; 32]; + + assert_eq!( + compute_bt_v1_file_cid(&ih1, 3), + compute_bt_v1_file_cid(&ih1, 3) + ); + assert_ne!( + compute_bt_v1_file_cid(&ih1, 0), + compute_bt_v1_file_cid(&ih1, 1) + ); + assert!(compute_bt_v1_file_cid(&ih1, 0).starts_with('b')); + assert!(compute_bt_v2_file_cid(&ih2, 0).starts_with('b')); + assert_ne!( + compute_bt_v1_file_cid(&[0xAB; 20], 0), + compute_bt_v2_file_cid(&[0xAB; 32], 0) + ); + } + + #[test] + fn nzb_release_cid_wire_and_properties() { + let cid = compute_nzb_release_cid("api.nzb.life", "485c52f078b580667a01ac34a1cef6c2"); + assert!(cid.starts_with('b')); + // Deterministic. + assert_eq!( + cid, + compute_nzb_release_cid("api.nzb.life", "485c52f078b580667a01ac34a1cef6c2") + ); + // Host-namespaced: same id, different host → different cid. + assert_ne!( + cid, + compute_nzb_release_cid("api.nzbgeek.info", "485c52f078b580667a01ac34a1cef6c2") + ); + // Explicit wire form: [01][84 20][84 20][20][sha256("host\nid")]. + let digest: [u8; 32] = + Sha256::digest(b"api.nzb.life\n485c52f078b580667a01ac34a1cef6c2").into(); + let mut want = vec![0x01, 0x84, 0x20, 0x84, 0x20, 0x20]; + want.extend_from_slice(&digest); + assert_eq!(cid, format!("b{}", base32_lower_no_padding(&want))); + // Distinct codec from the bt-file codecs in the 0x100x neighborhood. + assert_ne!(cid, compute_bt_v1_file_cid(&[0xAB; 20], 0)); + } + + #[test] + fn base32_known_vectors() { + assert_eq!(base32_lower_no_padding(b""), ""); + assert_eq!(base32_lower_no_padding(b"f"), "my"); + assert_eq!(base32_lower_no_padding(b"fo"), "mzxq"); + assert_eq!(base32_lower_no_padding(b"foo"), "mzxw6"); + assert_eq!(base32_lower_no_padding(b"foob"), "mzxw6yq"); + assert_eq!(base32_lower_no_padding(b"foobar"), "mzxw6ytboi"); + } + + #[test] + fn ipfs_cid_single_leaf_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b"hello world"), + "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" + ); + } + + #[test] + fn ipfs_cid_empty_matches_kubo() { + assert_eq!( + compute_ipfs_cid(b""), + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + } + + #[test] + fn ipfs_cid_multi_leaf_zeros_stable() { + let mb_zeros = vec![0u8; 1024 * 1024]; + assert_eq!( + compute_ipfs_cid(&mb_zeros), + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + } + + #[test] + fn ipfs_blocks_single_leaf() { + let payload = b"hello world"; + let out = compute_ipfs_blocks(payload); + assert_eq!(out.root, compute_ipfs_cid(payload)); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].0, out.root); + assert_eq!(out.blocks[0].1.as_ref(), payload); + } + + #[test] + fn ipfs_blocks_multi_leaf() { + let mb_zeros = vec![0u8; 1024 * 1024]; + let out = compute_ipfs_blocks(&mb_zeros); + assert_eq!( + out.root, + "bafybeiadh3bekpwtewjvauqeucf7yzqrb3ixsxzltnuwed4pxangtpou6m" + ); + assert_eq!(out.blocks.len(), 5); + assert_eq!(out.blocks.last().unwrap().0, out.root); + for (_cid, block) in &out.blocks[..4] { + assert_eq!(block.len(), 256 * 1024); + assert!(block.iter().all(|&b| b == 0)); + } + } + + #[test] + fn ipfs_blocks_empty() { + let out = compute_ipfs_blocks(b""); + assert_eq!( + out.root, + "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" + ); + assert_eq!(out.blocks.len(), 1); + assert_eq!(out.blocks[0].1.len(), 0); + assert_eq!(out.blocks[0].0, out.root); + } +} diff --git a/crates/meta-feeder-sdk/src/lang.rs b/crates/meta-feeder-sdk/src/lang.rs new file mode 100644 index 0000000..3d306c0 --- /dev/null +++ b/crates/meta-feeder-sdk/src/lang.rs @@ -0,0 +1,81 @@ +//! Language-code normalization — ISO 639 alias folding, **not** filename +//! parsing. +//! +//! This lives apart from [`crate::filename_meta`] on purpose: it doesn't read a +//! release title, it canonicalizes a *language code* (an indexer's declared +//! language, a query's `languages:` filter, a title-derived stamp) onto the +//! single ISO 639-2/B alpha-3 vocabulary the rest of the stack compares against. +//! It is the one piece of the old `filename_meta` module that has no equivalent +//! in the `metamesh-plugin-filename-parser` plugin — because it isn't filename +//! parsing — so it stays in Rust while the title-parsing algorithm itself is +//! owned by the plugin. Callers in the **request hot path** (e.g. the torznab +//! indexer-language fan-out skip) need it synchronously, with no network hop. + +/// Normalize a language code to the ISO 639-2/B alpha-3 vocabulary the stack +/// emits (`fre`/`ger`/`chi`, **not** the 639-2/T `fra`/`deu`/`zho` nor the +/// 639-1 two-letter forms), lowercased. This is the space meta-share's +/// "languages I read" preference uses and that the title-derived language stamps +/// write, so an operator who declares an indexer's language as `fra`/`de`/`zh` +/// still lines up with a `fre`/`ger`/`chi` query filter. Unknown / +/// already-canonical codes pass through lowercased+trimmed unchanged. +pub fn normalize_lang_code(code: &str) -> String { + let c = code.trim().to_ascii_lowercase(); + let mapped = match c.as_str() { + // 639-2/T and 639-1 → 639-2/B for the codes that actually differ. + "fra" | "fr" => "fre", + "deu" | "de" => "ger", + "zho" | "zh" => "chi", + "nld" | "nl" => "dut", + "ell" | "el" => "gre", + "ron" | "ro" => "rum", + "slk" | "sk" => "slo", + "ces" | "cs" => "cze", + // 639-1 → 639-2 for the rest of the language-tag output space (these + // share the same /B and /T alpha-3, so only the two-letter form maps). + "en" => "eng", + "ja" => "jpn", + "it" => "ita", + "es" => "spa", + "pt" => "por", + "ko" => "kor", + "ru" => "rus", + "pl" => "pol", + "hu" => "hun", + "uk" => "ukr", + "bg" => "bul", + "sv" => "swe", + "no" => "nor", + "da" => "dan", + "fi" => "fin", + "tr" => "tur", + "ar" => "ara", + "hi" => "hin", + "ta" => "tam", + "te" => "tel", + "ml" => "mal", + "vi" => "vie", + "th" => "tha", + _ => return c, + }; + mapped.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_lang_maps_to_639_2b_vocabulary() { + // 639-2/T and 639-1 spellings collapse onto the /B forms the stack uses. + assert_eq!(normalize_lang_code("fra"), "fre"); + assert_eq!(normalize_lang_code("FR"), "fre"); + assert_eq!(normalize_lang_code("deu"), "ger"); + assert_eq!(normalize_lang_code("zh"), "chi"); + assert_eq!(normalize_lang_code("nld"), "dut"); + // Already-canonical / unknown codes pass through trimmed + lowercased. + assert_eq!(normalize_lang_code(" ENG "), "eng"); + assert_eq!(normalize_lang_code("fre"), "fre"); + assert_eq!(normalize_lang_code("mul"), "mul"); + assert_eq!(normalize_lang_code("xyz"), "xyz"); + } +} diff --git a/crates/meta-feeder-sdk/src/lib.rs b/crates/meta-feeder-sdk/src/lib.rs new file mode 100644 index 0000000..dd62603 --- /dev/null +++ b/crates/meta-feeder-sdk/src/lib.rs @@ -0,0 +1,49 @@ +//! `meta-feeder-sdk` — the source-agnostic foundation shared by the gateway +//! core and every feeder sidecar binary. +//! +//! A *feeder* implements [`FeederPlugin`] (find records, fetch bytes) for one +//! or more upstreams, and [`serve_feeders`] exposes them over the feeder HTTP +//! contract. The gateway core consumes that contract via its +//! `RemoteFeederPlugin` and keeps the libp2p wire, the bitswap blockstore, the +//! hashing-into-the-blockstore, and the meta-core store-back to itself. +//! +//! Deliberately libp2p-free and blockstore-free — see this crate's `Cargo.toml`. + +// `redb::Error` is a large enum; the cache wrappers (`cache.rs`) return it +// directly rather than boxing on every embedded-DB call. Boxing each would be +// pure noise for a single-process store, so silence the lint crate-wide. +#![allow(clippy::result_large_err)] + +/// pcs-convention default meta-core root a self-publishing feeder targets when +/// neither its dashboard config nor the `META_CORE_URL` env seed sets one. Keeps +/// the modular storage seam out of compose: a standard install just works; a +/// non-standard meta-core is set in the feeder's config UI. Matches the +/// `metacore-app` service the AppStore feeder apps ship on the shared network. +pub const DEFAULT_META_CORE_URL: &str = "http://metacore-app:9000"; + +pub mod cache; +pub mod common; +pub mod config; +pub mod enrich; +pub mod filename_meta; +pub mod hash; +pub mod lang; +pub mod meta_core; +pub mod plugin; +pub mod query; +pub mod query_eval; +pub mod serve; +pub mod types; + +pub use config::{ConfigField, ConfigSchema, FieldKind}; +pub use enrich::{EnrichTarget, EnrichmentConfig, Enricher}; +pub use meta_core::FeederStore; +pub use plugin::{ + upstream_id_field, ConfigError, FeederPlugin, HashKind, HashOutcome, PluginRegistry, +}; +pub use query::{GatewayQuery, GatewaySearchEvent, GatewayWireError, Negation, RangeFilter}; +pub use serve::{ + configure_plugins, router, serve_feeders, ComputeRequest, ComputeResponse, HashKindDto, + HealthResponse, ManifestResponse, OutcomeDto, PluginManifest, QueryRequest, QueryResponse, +}; +pub use types::{ByteStream, DiscoveryId, DiscoveryRecord, GatewayError, Hash, PluginHealth}; diff --git a/crates/meta-feeder-sdk/src/meta_core.rs b/crates/meta-feeder-sdk/src/meta_core.rs new file mode 100644 index 0000000..93cfe52 --- /dev/null +++ b/crates/meta-feeder-sdk/src/meta_core.rs @@ -0,0 +1,293 @@ +//! Lightweight meta-core write/read client for feeders that **self-publish**. +//! +//! Post-"smart feeder / dumb gateway" split: a feeder given a `META_CORE_URL` +//! writes its own base record into meta-core instead of returning it for the +//! gateway dispatcher to store. This keeps the gateway a pure aggregator + +//! request router; the feeder owns the full record lifecycle for its upstream. +//! +//! The write shape is **byte-for-byte the same** as the gateway's +//! `meta_core::build_metadata_body` (`crates/meta-gateway/src/meta_core.rs`): +//! `cid_*` fields collapse to the `cids/ = "true"` key-set, and a +//! `provenance` JSON blob is stamped. Keeping them identical means a record +//! written by the feeder is indistinguishable from one the dispatcher wrote — +//! the meta-share ingester, meta-watch, and ranking all behave the same. +//! +//! `PUT /api/metadata/{cid}` is merge/upsert (confirmed: meta-core +//! `SetMetadataFlat` sets only the fields in the payload, never deletes), so the +//! feeder's base write and the enrichment plugins' later `PATCH /meta/{cid}` +//! merges accumulate on the same `/file/{cid}` hash. + +use std::collections::BTreeMap; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::Deserialize; +use tracing::warn; + +use crate::plugin::{HashKind, HashOutcome}; +use crate::types::{DiscoveryRecord, Hash}; + +/// Provenance recorded on every gateway-acquired record. Mirrors the gateway +/// dispatcher's shape so feeder-written and dispatcher-written records are +/// interchangeable. +#[derive(Clone, Debug)] +pub struct Provenance { + pub upstream: String, + pub record_id: String, + /// Stable per-deployment id (the gateway peer this feeder backs, from + /// `META_GATEWAY_PEER_ID`, else the feeder hostname). Analytics group by it. + pub gateway_peer: String, + pub acquired_at: u64, +} + +impl Provenance { + /// Build a provenance stamp for `upstream`/`record_id`, stamping "now". + pub fn now(upstream: &str, record_id: &str, gateway_peer: &str) -> Self { + Provenance { + upstream: upstream.to_string(), + record_id: record_id.to_string(), + gateway_peer: gateway_peer.to_string(), + acquired_at: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + } + } +} + +/// JSON-stringified provenance value. Same key set + `"source":"gateway"` as the +/// gateway dispatcher (`crates/meta-gateway/src/meta_core.rs::provenance_to_json`). +pub fn provenance_to_json(p: &Provenance) -> String { + serde_json::to_string(&serde_json::json!({ + "source": "gateway", + "upstream": p.upstream, + "recordId": p.record_id, + "gatewayPeer": p.gateway_peer, + "acquiredAt": p.acquired_at, + })) + .expect("static JSON shape — serialisation is infallible") +} + +/// Build the flat meta-core PUT body from a [`DiscoveryRecord`] + provenance. +/// +/// Identical transform to the gateway dispatcher: any `cid_*` field becomes a +/// `cids/ = "true"` key-set member (the CID is self-describing — the +/// algorithm is the multicodec; see METADATA_KEYS.md §2/§14.13), everything else +/// passes through, and `provenance` is stamped last (overriding any plugin-set +/// value). +pub fn build_metadata_body( + record: &DiscoveryRecord, + provenance: &Provenance, +) -> BTreeMap { + let mut body: BTreeMap = BTreeMap::new(); + for (k, v) in &record.fields { + if k.starts_with("cid_") && !v.is_empty() { + body.insert(format!("cids/{v}"), "true".to_string()); + } else { + body.insert(k.clone(), v.clone()); + } + } + body.insert("provenance".to_string(), provenance_to_json(provenance)); + body +} + +/// `PUT /api/metadata/{cid}` — write (merge/upsert) the record body into +/// meta-core. `base_url` is the meta-core root (e.g. `http://…-core:9000`). +pub async fn put_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, + body: &BTreeMap, +) -> anyhow::Result<()> { + let url = format!("{}/api/metadata/{}", base_url.trim_end_matches('/'), cid); + let resp = http.put(&url).json(body).send().await?; + let status = resp.status(); + if !status.is_success() { + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("meta-core PUT {url} returned {status}: {text}"); + } + Ok(()) +} + +/// Shape of meta-core's `GET /meta/{cid}` response (`{ hashId, metadata }`). +#[derive(Debug, Deserialize)] +struct MetaGetResponse { + #[serde(default)] + metadata: Option>, +} + +/// `GET /meta/{cid}` — read the current flat field map for a record, or `None` +/// when meta-core has no record yet (404). Used to read back a plugin's merged +/// fields (e.g. filename-parser's `originalTitle`/`movieYear`) before handing +/// them to the next plugin in the pipeline. +pub async fn get_record( + http: &reqwest::Client, + base_url: &str, + cid: &str, +) -> anyhow::Result>> { + let url = format!("{}/meta/{}", base_url.trim_end_matches('/'), cid); + let resp = http.get(&url).send().await?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !resp.status().is_success() { + anyhow::bail!("meta-core GET {url} returned {}", resp.status()); + } + let parsed: MetaGetResponse = resp.json().await?; + Ok(parsed.metadata) +} + +/// Self-publishing store for feeders that own their record lifecycle (D4): +/// the feeder writes its own **metadata-only** records to meta-core and returns +/// sparse outcomes so the gateway dispatcher skips them; the meta-share ingester +/// Kamilata-propagates them. +/// +/// **Byte outcomes are deliberately NOT self-published.** A feeder-side WebDAV +/// write to the gateway core would be seeded by nobody: the gateway core runs +/// with `ENABLE_FILE_WATCHER=false` (so meta-core emits no `/api/events/files` +/// for it, and the ingester's `ipfs_seed` — which seeds only watched files — +/// never sees it), and a sparse outcome makes the gateway skip its own +/// blockstore seed. So byte-bearing outcomes (torznab full-file / subtitles) +/// pass through unchanged and the **gateway** continues to WebDAV-PUT + bitswap- +/// seed them (it's the libp2p/bitswap host). Result: feeder owns the metadata +/// records (the Kamilata-propagated bulk), gateway owns byte delivery — a clean +/// split with no seeding regression. Tribler is metadata-only and uses +/// [`crate::enrich::Enricher`]; this is torznab's record-owning counterpart. +#[derive(Clone)] +pub struct FeederStore { + http: reqwest::Client, + meta_core_url: String, + gateway_peer: String, +} + +impl FeederStore { + pub fn new(http: reqwest::Client, meta_core_url: String, gateway_peer: String) -> Self { + FeederStore { + http, + meta_core_url: meta_core_url.trim_end_matches('/').to_string(), + gateway_peer, + } + } + + /// Build from the feeder env. `None` when `META_CORE_URL` is unset (the + /// feeder then falls back to the legacy "return outcomes, gateway stores" + /// path). `gateway_peer` ← `META_GATEWAY_PEER_ID` / `HOSTNAME`. + pub fn from_env() -> Option { + let meta_core_url = env_nonempty("META_CORE_URL")?; + let gateway_peer = env_nonempty("META_GATEWAY_PEER_ID") + .or_else(|| env_nonempty("HOSTNAME")) + .unwrap_or_else(|| "gateway-feeder".to_string()); + let http = crate::common::build_http_client(30, "meta-feeder-store", None); + Some(Self::new(http, meta_core_url, gateway_peer)) + } + + pub fn meta_core_url(&self) -> &str { + &self.meta_core_url + } + + /// Self-publish a **metadata-only** outcome to meta-core, returning the + /// **sparse** outcome (`record: None`) so the gateway's `(_, None)` branch + /// skips it — no double write. Byte outcomes and already-sparse outcomes are + /// returned unchanged (the gateway stores + seeds the bytes; see the struct + /// docs). On a write failure the original outcome is returned so the gateway + /// stores it (graceful degradation, never lost). + pub async fn publish( + &self, + upstream_id: &str, + record_id: &str, + outcome: HashOutcome, + ) -> HashOutcome { + // Only self-publish metadata-only outcomes (bytes None, record Some). + if outcome.bytes.is_some() || outcome.record.is_none() { + return outcome; + } + let cid = outcome.hash.0.clone(); + let record = outcome.record.as_ref().expect("record is Some by the guard"); + match self + .store_metadata_only(upstream_id, record_id, &cid, record) + .await + { + Ok(()) => sparse(outcome.hash, outcome.hash_kind), + Err(e) => { + warn!(target: "meta-feeder::store", cid = %cid, error = %e, + "metadata self-publish failed; falling back to gateway store"); + outcome + } + } + } + + /// Metadata-only self-publish: `PUT /api/metadata/{cid}` with provenance. + async fn store_metadata_only( + &self, + upstream_id: &str, + record_id: &str, + cid: &str, + record: &DiscoveryRecord, + ) -> anyhow::Result<()> { + let prov = Provenance::now(upstream_id, record_id, &self.gateway_peer); + let body = build_metadata_body(record, &prov); + put_record(&self.http, &self.meta_core_url, cid, &body).await + } +} + +fn sparse(hash: Hash, hash_kind: HashKind) -> HashOutcome { + HashOutcome { + hash, + hash_kind, + bytes: None, + record: None, + file_extension: None, + } +} + +fn env_nonempty(key: &str) -> Option { + std::env::var(key) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rec() -> DiscoveryRecord { + DiscoveryRecord { + upstream_id: "tribler".into(), + record_id: "abc".into(), + fields: BTreeMap::from_iter([ + ("title".into(), "Big Buck Bunny".into()), + ("cid_btih_v1_file".into(), "bafkPERFILE".into()), + ]), + } + } + + #[test] + fn provenance_shape_matches_gateway() { + let p = Provenance { + upstream: "tribler".into(), + record_id: "abc".into(), + gateway_peer: "metagateway-tribler".into(), + acquired_at: 1_747_504_000, + }; + let v: serde_json::Value = serde_json::from_str(&provenance_to_json(&p)).unwrap(); + assert_eq!(v["source"], "gateway"); + assert_eq!(v["upstream"], "tribler"); + assert_eq!(v["recordId"], "abc"); + assert_eq!(v["gatewayPeer"], "metagateway-tribler"); + assert_eq!(v["acquiredAt"], 1_747_504_000_u64); + } + + #[test] + fn body_collapses_cid_fields_and_stamps_provenance() { + let p = Provenance::now("tribler", "abc", "peer"); + let body = build_metadata_body(&rec(), &p); + assert_eq!(body.get("title").map(String::as_str), Some("Big Buck Bunny")); + // cid_* collapsed to cids/=true; no cid_* survives. + assert!(body.keys().all(|k| !k.starts_with("cid_"))); + assert_eq!( + body.get("cids/bafkPERFILE").map(String::as_str), + Some("true") + ); + assert!(body.contains_key("provenance")); + } +} diff --git a/crates/meta-feeder-sdk/src/plugin.rs b/crates/meta-feeder-sdk/src/plugin.rs new file mode 100644 index 0000000..84413b1 --- /dev/null +++ b/crates/meta-feeder-sdk/src/plugin.rs @@ -0,0 +1,234 @@ +//! The `FeederPlugin` trait + supporting types. +//! +//! A feeder plugin bridges one external upstream (`gutenberg`, `arxiv`, +//! `torznab`, …) into the MetaMesh network. It **finds** records and +//! **fetches** bytes; it does NOT hash-into-a-blockstore, store back into +//! meta-core, or speak libp2p — those stay in the gateway core. This is why +//! `FeederPlugin` has no `set_blockstore` / `set_tmdb_budget` hooks (the +//! gateway's `GatewayPlugin` carried them): preview-seeding moves to the core, +//! and a feeder that needs a TMDB budget owns it internally. +//! +//! The `serve_feeders` harness ([`crate::serve`]) wraps any set of +//! `FeederPlugin`s in the feeder HTTP contract; the gateway core's +//! `RemoteFeederPlugin` is the matching HTTP client. + +use std::collections::HashMap; +use std::path::Path; + +use async_trait::async_trait; +use futures::stream::BoxStream; + +// Re-exported so plugins can `use meta_feeder_sdk::plugin::GatewayQuery` (the +// gateway's original `plugin.rs` re-exported it the same way). +pub use crate::query::GatewayQuery; +use crate::query::GatewaySearchEvent; +use crate::types::{ByteStream, DiscoveryRecord, GatewayError, Hash, PluginHealth}; + +/// One element of what [`FeederPlugin::compute_outcomes`] returns. +/// +/// The `hash` is the value shipped to the gateway core. The other fields are +/// signals the core's auto-store uses to persist the content into meta-core. +/// The core's three-branch auto-store routes on `(bytes, record)`: +/// - `(Some, Some)` — full store: WebDAV PUT + metadata PUT with `filePath`. +/// - `(None, Some)` — metadata-only PUT (no local bytes; e.g. torznab samples +/// the middle 1 MiB of a multi-GiB video and surfaces only the record). +/// - `(_, None)` — skip. Cache hits land here. +/// +/// `file_extension` MUST be `None` when `bytes` is `None`. +#[derive(Debug)] +pub struct HashOutcome { + pub hash: Hash, + /// Which hash family this CID belongs to. Drives the core's routing — + /// the bitswap blockstore seed only fires for `Sha2_256`. + pub hash_kind: HashKind, + pub bytes: Option, + pub record: Option, + /// File extension to append when naming the WebDAV-side blob (e.g. + /// `"epub"`). No leading dot. MUST be `None` when `bytes` is `None`. + pub file_extension: Option, +} + +/// Hash family discriminator on [`HashOutcome`]. +/// +/// - `Midhash256` — size-prefix-plus-middle-1MB-sample, custom multicodec +/// `0x1000`. Fast, MetaMesh-internal, not retrievable via public IPFS. +/// - `Sha2_256` — standard IPFS CIDv1 over full bytes. Retrievable via bitswap +/// once the bytes are in the core's blockstore. What every shipping plugin +/// today emits. +/// - `BtV1File` — a single file inside a BitTorrent v1 torrent, custom +/// multicodec `0x1001`. Locator, not content hash: opaque to bitswap. +/// - `NzbRelease` — a Usenet release (a Newznab listing), custom multicodec +/// `nzb-release` `0x1004`. Locator like `BtV1File`: opaque to bitswap, +/// redeemed to bytes only by a credentialed meta-share peer, which grabs the +/// `.nzb` via the `{base, id}` descriptor stored at `nzb:` KV. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashKind { + /// Reserved: no feeder currently emits midhash256 outcomes, but the family + /// is real across meta-core / meta-share, so the variant stays. + #[allow(dead_code)] + Midhash256, + Sha2_256, + BtV1File, + NzbRelease, +} + +/// Startup-only configuration error returned by [`FeederPlugin::configure`]. +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("plugin {plugin}: required config missing: {what}")] + MissingConfig { + plugin: &'static str, + /// Human-readable description of the missing setting and where to + /// supply it. Used by the harness's soft-skip warning. + what: &'static str, + }, + + #[error("plugin {plugin}: cache dir setup failed: {source}")] + CacheSetup { + plugin: &'static str, + #[source] + source: std::io::Error, + }, + + #[error("plugin {plugin}: {source}")] + Other { + plugin: &'static str, + #[source] + source: anyhow::Error, + }, +} + +/// Canonical field name for an upstream's stable per-record identifier. +/// Every plugin's `DiscoveryRecord.fields` MUST include this key +/// (`gutenbergid:11`, `tmdbid:31910`, `arxivid:2106.07447`). The rule is +/// `format!("{upstream_id}id")` — lowercase, no separator. +pub fn upstream_id_field(upstream_id: &str) -> String { + format!("{upstream_id}id") +} + +/// Key-set prefix for the provenance `source/