From 8a8c02c97e8d1dd7787828676b77caa38a53a5b5 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 24 Aug 2026 19:13:08 +0200 Subject: [PATCH] feeder rework --- .gitattributes | 6 + .github/workflows/publish.yml | 59 + Cargo.lock | 1956 +++++++++++++++++++++++ Cargo.toml | 9 + feeder-plugin/arxiv-feeder/Cargo.toml | 40 + feeder-plugin/arxiv-feeder/Dockerfile | 44 + feeder-plugin/arxiv-feeder/src/arxiv.rs | 845 ++++++++++ feeder-plugin/arxiv-feeder/src/lib.rs | 4 + feeder-plugin/arxiv-feeder/src/main.rs | 30 + rust-toolchain.toml | 3 + 10 files changed, 2996 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/publish.yml create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 feeder-plugin/arxiv-feeder/Cargo.toml create mode 100644 feeder-plugin/arxiv-feeder/Dockerfile create mode 100644 feeder-plugin/arxiv-feeder/src/arxiv.rs create mode 100644 feeder-plugin/arxiv-feeder/src/lib.rs create mode 100644 feeder-plugin/arxiv-feeder/src/main.rs create mode 100644 rust-toolchain.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9f43494 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# FIX CRLF always when developer has not set +# Linux/Mac: git config --global core.autocrlf input +# Windows: git config --global core.autocrlf true +# Auto detect text files and perform LF normalization +* text=auto +* eol=lf diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..c89c02c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,59 @@ +name: Publish image + +# Build and push the feeder image to the GitHub Container Registry (ghcr.io): +# * every push to main -> the moving `main` tag +# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1) +# The build context is the repo root (the workspace) so the vendored +# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under +# feeder-plugin/paper-feeder/. +on: + push: + branches: [main] + tags: ['v*'] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=sha,format=short + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./feeder-plugin/arxiv-feeder/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..e003869 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1956 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arxiv-feeder" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "futures", + "meta-feeder-sdk", + "quick-xml", + "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "tracing-subscriber", + "wiremock", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "meta-feeder-sdk" +version = "0.1.0" +source = "git+https://github.com/worph/meta-feeder-sdk?rev=d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4#d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" +dependencies = [ + "anyhow", + "async-trait", + "axum", + "base64", + "bytes", + "futures", + "hyper", + "redb", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.36.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redb" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" +dependencies = [ + "libc", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..5dfe0ba --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[workspace] +resolver = "2" +members = ["feeder-plugin/*"] + +[workspace.package] +edition = "2021" +rust-version = "1.85" +license = "MIT" +repository = "https://github.com/worph/meta-feeder-arxiv" diff --git a/feeder-plugin/arxiv-feeder/Cargo.toml b/feeder-plugin/arxiv-feeder/Cargo.toml new file mode 100644 index 0000000..56064c8 --- /dev/null +++ b/feeder-plugin/arxiv-feeder/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "arxiv-feeder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +# arXiv preprints. Open, keyless, no credential. +# Upstream: arXiv (export.arxiv.org) + +[lib] +name = "arxiv_feeder" +path = "src/lib.rs" + +[[bin]] +name = "arxiv-feeder" +path = "src/main.rs" + +[dependencies] +# The SDK is a dependency, not a vendored copy. Pinned by rev until the first +# tag; see docs/project-architecture/feeder-architecture.md §8. +meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" } + +tokio = { version = "1.40", features = ["full"] } +axum = { version = "0.7", features = ["json"] } +anyhow = "1" +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false } +futures = "0.3" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +quick-xml = "0.36" + +[dev-dependencies] +wiremock = "0.6" +tempfile = "3" +tokio = { version = "1.40", features = ["full", "test-util"] } diff --git a/feeder-plugin/arxiv-feeder/Dockerfile b/feeder-plugin/arxiv-feeder/Dockerfile new file mode 100644 index 0000000..aa1095c --- /dev/null +++ b/feeder-plugin/arxiv-feeder/Dockerfile @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# arxiv-feeder sidecar. Multi-stage build; build context is this repo's root. +# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied +# in — `git` must be present in the builder for cargo to fetch it. + +FROM rust:1.89-slim-bookworm AS builder +WORKDIR /build + +RUN apt-get update && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml rust-toolchain.toml ./ +COPY feeder-plugin/ feeder-plugin/ + +RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ + --mount=type=cache,id=arxiv-feeder-target,target=/build/target \ + cargo build --release --bin arxiv-feeder \ + && cp target/release/arxiv-feeder /arxiv-feeder + +# ----------------------------------------------------------------------------- +FROM debian:bookworm-slim AS runtime + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /arxiv-feeder /usr/local/bin/arxiv-feeder + +RUN useradd --system --uid 10001 --no-create-home metafeeder \ + && mkdir -p /data/meta-feeder \ + && chown metafeeder:metafeeder /data/meta-feeder + +USER metafeeder +WORKDIR /data/meta-feeder + +EXPOSE 8080 + +ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \ + META_FEEDER_STATE_DIR=/data/meta-feeder \ + RUST_LOG=info + +ENTRYPOINT ["/usr/local/bin/arxiv-feeder"] diff --git a/feeder-plugin/arxiv-feeder/src/arxiv.rs b/feeder-plugin/arxiv-feeder/src/arxiv.rs new file mode 100644 index 0000000..8b0f8a2 --- /dev/null +++ b/feeder-plugin/arxiv-feeder/src/arxiv.rs @@ -0,0 +1,845 @@ +//! arXiv bridge via the public [arXiv API](https://info.arxiv.org/help/api/index.html). +//! +//! - Search: `GET /api/query?search_query=all:&max_results=N` → Atom feed. +//! - Fetch one: `GET /api/query?id_list=` → Atom feed (single entry). +//! - Canonical file format for midhash: the entry's PDF link +//! (``). arXiv always provides one; fall back +//! to the derived URL pattern `/pdf/.pdf` if the link +//! is missing for some reason. +//! +//! No auth, no API key. arXiv asks for a 3-second courtesy delay between +//! search requests — we don't poll automatically (only on user search + +//! on-demand midhash compute), so we satisfy that without explicit rate +//! limiting. +//! +//! **Versioned record_ids.** arXiv search returns ids like +//! `2106.07447v1` — the version suffix matters because v2 of the same +//! paper is a different file with a different midhash. The plugin +//! preserves whatever record_id the search returned and uses it +//! verbatim for the PDF fetch, so the (record_id → midhash) mapping is +//! stable across re-computes. +//! +//! See `docs/gateway-feature.md` §4 for the v0 plugin role. + +use std::collections::BTreeMap; +use std::path::Path; + +use async_trait::async_trait; +use quick_xml::events::Event; +use quick_xml::Reader; + +use meta_feeder_sdk::common; +use meta_feeder_sdk::cache::MidhashCache; +#[cfg(test)] +use meta_feeder_sdk::plugin::HashKind; +use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, GatewayQuery, HashOutcome}; +use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth}; + +/// Public arXiv API base. `export.arxiv.org` is the recommended host +/// per arXiv's API docs (it's the rate-limit-friendly mirror; also +/// serves `/pdf/` so we get both API and PDF off the same host — +/// simpler test setup). +const DEFAULT_BASE_URL: &str = "https://export.arxiv.org"; + +/// Generous HTTP timeout. arXiv responds in <1 s usually but PDF +/// downloads of larger papers can hit 10–20 s on cold caches. +const HTTP_TIMEOUT_SECS: u64 = 30; + +/// Polite identification per the arXiv API guidelines. +const USER_AGENT: &str = concat!("meta-share/", env!("CARGO_PKG_VERSION"), " (gateway:arxiv)"); + +/// arXiv gateway plugin. Same structural shape as gutenberg: cheap to +/// construct; `configure()` opens the per-plugin redb cache. The cache +/// stores `(record_id → midhash)` so we don't re-download a multi-MB +/// PDF on every "add" click. +pub struct ArxivPlugin { + http: reqwest::Client, + base_url: String, + cache: Option, +} + +impl Default for ArxivPlugin { + fn default() -> Self { + Self::new() + } +} + +impl ArxivPlugin { + pub fn new() -> Self { + Self::with_base_url(DEFAULT_BASE_URL.to_string()) + } + + /// Construct the plugin pointing at a non-default base URL — used by + /// tests (against a `wiremock::MockServer`) and by deployments that + /// want to ride a private mirror. The base must serve both + /// `/api/query` (Atom feed) and `/pdf/.pdf` paths. + pub fn with_base_url(base_url: String) -> Self { + let http = common::build_http_client(HTTP_TIMEOUT_SECS, USER_AGENT, None); + Self { + http, + base_url, + cache: None, + } + } + + fn cache(&self) -> Result<&MidhashCache, GatewayError> { + common::require_cache(self.cache.as_ref(), "arxiv") + } + + /// Search the arXiv Atom API. `query` is passed verbatim as the + /// `all:` field, so callers get the full-text behaviour arXiv users + /// expect (free-text + boolean syntax both work upstream). + async fn search_atom(&self, query: &str, max_results: usize) -> Result, GatewayError> { + let url = format!( + "{}/api/query?search_query=all:{}&start=0&max_results={}", + self.base_url.trim_end_matches('/'), + common::urlencode(query), + max_results + ); + common::fetch_bytes(&self.http, &url) + .await + .map(|b| b.to_vec()) + } + + /// Single-entry lookup by id. Used by `compute_outcomes` to refresh + /// the record metadata + locate the PDF link. Same Atom shape as + /// search; the `id_list=` form returns at most one entry. + async fn fetch_entry(&self, record_id: &str) -> Result { + let url = format!( + "{}/api/query?id_list={}", + self.base_url.trim_end_matches('/'), + common::urlencode(record_id) + ); + let body = common::fetch_bytes(&self.http, &url).await?; + let mut entries = parse_arxiv_feed(&body).map_err(|e| { + GatewayError::Permanent(format!("parse arxiv atom for {record_id}: {e}")) + })?; + if entries.is_empty() { + return Err(GatewayError::NotFound); + } + Ok(entries.remove(0)) + } + + /// Decide which URL to fetch for `compute_outcomes`. The entry's + /// `` is preferred (canonical, version-stable); + /// falls back to a derived `/pdf/.pdf` pattern if absent. The + /// fallback also rewrites scheme/host to the configured base so a + /// test wiremock can intercept (real arXiv emits absolute URLs + /// pointing at `arxiv.org`, which would bypass a wiremock pointed + /// at a different host). + fn pdf_url_for(&self, entry: &ArxivEntry, record_id: &str) -> String { + let base = self.base_url.trim_end_matches('/'); + if let Some(link) = entry.pdf_link.as_deref() { + // For real arXiv: the link absolute-URL points at + // `arxiv.org`. For private mirrors / wiremock: the link + // already points at our base. Rewrite the host iff our base + // host differs from the link's, so the same code works in + // both cases. + if let Some(rewritten) = rewrite_host(link, base) { + return rewritten; + } + } + format!("{base}/pdf/{record_id}.pdf") + } +} + +#[async_trait] +impl FeederPlugin for ArxivPlugin { + fn upstream_id(&self) -> &'static str { + "arxiv" + } + + fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> { + self.cache = Some(common::open_midhash_cache(cache_dir, "arxiv")?); + Ok(()) + } + + async fn handle_query( + &self, + query: &GatewayQuery, + max_results: usize, + ) -> Result, GatewayError> { + // Layer A early-return: arXiv only serves `document` / + // `paper`. A `fileType:image` or `contentKind:movie` filter + // can never match — skip the upstream call entirely. + if !meta_feeder_sdk::query_eval::query_accepts_plugin( + query, + self.served_file_types(), + self.served_content_kinds(), + ) { + return Ok(Vec::new()); + } + let q = query.free_text_or_star(); + let body = self.search_atom(q, max_results).await?; + let entries = parse_arxiv_feed(&body).map_err(|e| { + GatewayError::Permanent(format!("parse arxiv atom search response: {e}")) + })?; + // arXiv's `max_results=` is a soft cap; truncate defensively so + // we never return more than the caller asked for. + Ok(entries + .into_iter() + .take(max_results) + .map(into_discovery_record) + .collect()) + } + + async fn compute_outcomes(&self, record_id: &str) -> Result, GatewayError> { + let cache = self.cache()?; + if let Some(hit) = common::cached_outcome(cache, record_id, "arxiv")? { + return Ok(hit); + } + + let entry = self.fetch_entry(record_id).await?; + let pdf_url = self.pdf_url_for(&entry, record_id); + let bytes = common::fetch_bytes(&self.http, &pdf_url).await?; + let cid = meta_feeder_sdk::hash::compute_ipfs_cid(&bytes); + + common::store_midhash(cache, record_id, "arxiv", &cid); + + let record = into_discovery_record(entry); + Ok(common::single_outcome( + cid, + bytes, + record, + Some("pdf".to_string()), + )) + } + + fn health(&self) -> PluginHealth { + if self.cache.is_some() { + PluginHealth::Ok + } else { + PluginHealth::Degraded { + reason: "configure() not yet called".to_string(), + } + } + } + + fn served_file_types(&self) -> &'static [&'static str] { + &["document"] + } + + fn served_content_kinds(&self) -> &'static [&'static str] { + &["paper"] + } +} + +// -- Atom feed parsing ------------------------------------------------------- +// +// arXiv's API returns an Atom feed with one `` per result. We +// only need a handful of fields; the parser walks events and accumulates +// per-entry state rather than building a full DOM. quick-xml's +// streaming API is ~10x lighter than serde-xml here. + +/// Parsed shape of one Atom `` element. Fields the gateway +/// surfaces as `DiscoveryRecord.fields` after the +/// `into_discovery_record` conversion. +#[derive(Debug, Default, PartialEq)] +struct ArxivEntry { + /// Canonical record_id (the `2106.07447v1`-style suffix of the + /// `http://arxiv.org/abs/...` element). Versioned when the + /// search returned a versioned id; unversioned otherwise. Stable + /// across re-computes. + record_id: String, + title: Option, + summary: Option, + /// ISO 8601 `published` timestamp, kept verbatim — used for + /// the `year` derived field and as a sortable string in meta-core. + published: Option, + authors: Vec, + /// All `` values. arXiv uses the `cs.AI`, + /// `math.GT`, etc. taxonomy. Stored as a Vec; collapsed to a CSV + /// when emitting the DiscoveryRecord. + categories: Vec, + /// ``. Always present in arXiv's + /// feed; may equal `categories[0]` or not, depending on tagging. + primary_category: Option, + /// Absolute URL of the PDF, extracted from + /// ``. Used directly by + /// `compute_outcomes` (the version suffix in the URL is what makes + /// the midhash stable across re-computes). + pdf_link: Option, +} + +/// Parse an arXiv Atom feed into entries. Tolerant of namespace +/// prefixes (`arxiv:primary_category`) and self-closing `` tags. +fn parse_arxiv_feed(xml: &[u8]) -> Result, quick_xml::Error> { + let mut reader = Reader::from_reader(xml); + reader.config_mut().trim_text(true); + + let mut entries: Vec = Vec::new(); + let mut in_entry = false; + let mut current: ArxivEntry = ArxivEntry::default(); + // Track which simple text field we're collecting between open + close. + // Only one is open at a time within an entry. + let mut field: TextField = TextField::None; + // Author name extraction: we enter ``, then ``, then + // capture text. `in_author_name` is true only inside `` inside + // ``. + let mut in_author = false; + let mut in_author_name = false; + let mut text_buf = String::new(); + let mut event_buf = Vec::with_capacity(1024); + + loop { + match reader.read_event_into(&mut event_buf)? { + Event::Start(e) => { + let name = local_name(e.name().as_ref()); + if name == "entry" { + in_entry = true; + current = ArxivEntry::default(); + } else if in_entry { + match name.as_str() { + "id" => field = TextField::Id, + "title" => field = TextField::Title, + "summary" => field = TextField::Summary, + "published" => field = TextField::Published, + "author" => in_author = true, + "name" if in_author => { + in_author_name = true; + field = TextField::Author; + } + _ => {} + } + } + } + Event::Empty(e) => { + // Self-closing tags: link, category, arxiv:primary_category. + if !in_entry { + continue; + } + let name = local_name(e.name().as_ref()); + match name.as_str() { + "link" => { + let attrs = collect_attrs(&e); + // arXiv emits two links per entry: rel=alternate + // (HTML abs page) and rel=related title=pdf. We + // only want the PDF. + if attrs.get("title").map(String::as_str) == Some("pdf") { + if let Some(href) = attrs.get("href") { + current.pdf_link = Some(href.clone()); + } + } + } + "category" => { + let attrs = collect_attrs(&e); + if let Some(term) = attrs.get("term") { + current.categories.push(term.clone()); + } + } + "primary_category" => { + let attrs = collect_attrs(&e); + if let Some(term) = attrs.get("term") { + current.primary_category = Some(term.clone()); + } + } + _ => {} + } + } + Event::Text(t) => { + if !in_entry { + continue; + } + match field { + TextField::None => {} + _ => { + // Atom feeds may split text across events when + // entity references are involved; accumulate. + if let Ok(s) = t.unescape() { + text_buf.push_str(&s); + } + } + } + } + Event::End(e) => { + let name = local_name(e.name().as_ref()); + if !in_entry && name != "entry" { + continue; + } + match name.as_str() { + "id" => { + current.record_id = strip_abs_prefix(&text_buf); + finish_field(&mut field, &mut text_buf); + } + "title" => { + current.title = Some(normalize_whitespace(&text_buf)); + finish_field(&mut field, &mut text_buf); + } + "summary" => { + current.summary = Some(normalize_whitespace(&text_buf)); + finish_field(&mut field, &mut text_buf); + } + "published" => { + current.published = Some(text_buf.trim().to_string()); + finish_field(&mut field, &mut text_buf); + } + "name" if in_author_name => { + let name = text_buf.trim().to_string(); + if !name.is_empty() { + current.authors.push(name); + } + in_author_name = false; + finish_field(&mut field, &mut text_buf); + } + "author" => { + in_author = false; + } + "entry" => { + in_entry = false; + entries.push(std::mem::take(&mut current)); + } + _ => {} + } + } + Event::Eof => break, + _ => {} + } + event_buf.clear(); + } + Ok(entries) +} + +/// Which simple text-bearing field the parser is currently inside. +/// Author names live inside `...`, so the +/// state machine flips `Author` only when both flags are set. +#[derive(Debug)] +enum TextField { + None, + Id, + Title, + Summary, + Published, + Author, +} + +fn finish_field(field: &mut TextField, buf: &mut String) { + *field = TextField::None; + buf.clear(); +} + +/// quick-xml hands element names with any XML-namespace prefix attached +/// (e.g. `arxiv:primary_category`). We treat both prefixed and bare +/// forms the same — arXiv only uses one namespace beyond Atom, and we +/// match against the local-name suffix only. +fn local_name(raw: &[u8]) -> String { + let s = std::str::from_utf8(raw).unwrap_or_default(); + match s.rfind(':') { + Some(i) => s[i + 1..].to_string(), + None => s.to_string(), + } +} + +/// Walk a `Start`/`Empty` event's attributes into a `key → value` map. +/// Small allocations only; called once per `` / ``. +fn collect_attrs(e: &quick_xml::events::BytesStart<'_>) -> BTreeMap { + let mut out = BTreeMap::new(); + for attr in e.attributes().flatten() { + let key = local_name(attr.key.as_ref()); + let val = std::str::from_utf8(&attr.value) + .unwrap_or_default() + .to_string(); + out.insert(key, val); + } + out +} + +/// `http://arxiv.org/abs/2106.07447v1` → `2106.07447v1`. +/// Tolerates `https://` and missing trailing version. Returns the +/// input verbatim if it doesn't look like an arXiv abs URL. +fn strip_abs_prefix(raw: &str) -> String { + let trimmed = raw.trim(); + for prefix in [ + "http://arxiv.org/abs/", + "https://arxiv.org/abs/", + "http://export.arxiv.org/abs/", + "https://export.arxiv.org/abs/", + ] { + if let Some(rest) = trimmed.strip_prefix(prefix) { + return rest.to_string(); + } + } + trimmed.to_string() +} + +/// Collapse runs of whitespace into single spaces and trim. arXiv +/// titles and summaries come with line-wrapping baked in; the UI +/// renders them in a single line so we normalize at ingest. +fn normalize_whitespace(raw: &str) -> String { + raw.split_whitespace().collect::>().join(" ") +} + +/// Rewrite the host of an absolute URL so a test wiremock can intercept +/// it. Returns `None` if the input isn't a valid absolute http(s) URL. +/// Real arXiv emits PDF links pointing at `arxiv.org` even when the +/// API is hit through `export.arxiv.org`; this lets us always fetch +/// the PDF through the same host we already configured for search. +fn rewrite_host(url: &str, new_base: &str) -> Option { + let url = url.trim(); + let (_scheme, rest_with_path) = if let Some(s) = url.strip_prefix("http://") { + ("http", s) + } else if let Some(s) = url.strip_prefix("https://") { + ("https", s) + } else { + return None; + }; + let path_start = rest_with_path.find('/')?; + let path = &rest_with_path[path_start..]; + Some(format!("{}{path}", new_base.trim_end_matches('/'))) +} + +/// Percent-encode a query-string value. arXiv accepts a small alphabet +/// in `search_query=` and `id_list=`; the safe-set covers everything +/// the gateway emits (free-text queries, dotted arXiv ids, version +/// suffixes). +/// Convert a parsed arXiv entry into the wire-shape `DiscoveryRecord`. +/// Field naming follows `docs/gateway-feature.md` §6.4 (`title`, `type`, +/// `sourceUrl`, `fileName`) with arXiv-specific keys (`arxivid`, +/// `author`, `categories`, `primaryCategory`, `summary`, `published`). +/// The canonical `arxivid` field is required (§6.4 — `format!("{upstream_id}id")`), +/// so typed search filters like `arxivid:2106.07447` work. +fn into_discovery_record(entry: ArxivEntry) -> DiscoveryRecord { + let record_id = entry.record_id.clone(); + let mut fields: BTreeMap = BTreeMap::new(); + + if let Some(t) = entry.title { + fields.insert("title".to_string(), t); + } + fields.insert("fileType".to_string(), "document".to_string()); + fields.insert("contentKind".to_string(), "paper".to_string()); + fields.insert( + "sourceUrl".to_string(), + format!("https://arxiv.org/abs/{record_id}"), + ); + fields.insert("fileName".to_string(), format!("arxiv-{record_id}.pdf")); + fields.insert(upstream_id_field("arxiv"), record_id.clone()); + fields.insert("format".to_string(), "pdf".to_string()); + if !entry.authors.is_empty() { + fields.insert("author".to_string(), entry.authors.join(", ")); + } + if !entry.categories.is_empty() { + fields.insert("categories".to_string(), entry.categories.join(", ")); + } + if let Some(pc) = entry.primary_category { + fields.insert("primaryCategory".to_string(), pc); + } + if let Some(p) = entry.published.as_deref() { + fields.insert("published".to_string(), p.to_string()); + // arXiv's `published` is ISO 8601 like `2021-06-14T17:46:43Z` — + // first 4 chars are the year. Useful for the UI's year filter + // and meta-core's structured tokenization. + if p.len() >= 4 && p[..4].chars().all(|c| c.is_ascii_digit()) { + fields.insert("year".to_string(), p[..4].to_string()); + } + } + if let Some(s) = entry.summary { + fields.insert("summary".to_string(), s); + } + + DiscoveryRecord { + upstream_id: "arxiv".to_string(), + record_id, + fields, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SAMPLE_FEED: &str = r#" + + ArXiv Query: search_query=all:test + 2026-01-01T00:00:00Z + + http://arxiv.org/abs/2106.07447v1 + 2021-06-15T18:00:00Z + 2021-06-14T17:46:43Z + HuBERT: Self-Supervised Speech Representation Learning + We propose a self-supervised approach. The model learns from data. + Wei-Ning Hsu + Benjamin Bolte + + + + + + + + http://arxiv.org/abs/1706.03762v5 + Attention Is All You Need + The dominant sequence transduction models are based on RNNs. + Ashish Vaswani + Noam Shazeer + 2017-06-12T17:57:34Z + + + + + +"#; + + #[test] + fn parses_atom_feed_into_entries() { + let entries = parse_arxiv_feed(SAMPLE_FEED.as_bytes()).expect("parse"); + assert_eq!(entries.len(), 2); + + let e0 = &entries[0]; + assert_eq!(e0.record_id, "2106.07447v1"); + assert_eq!( + e0.title.as_deref(), + Some("HuBERT: Self-Supervised Speech Representation Learning") + ); + assert_eq!(e0.authors, vec!["Wei-Ning Hsu", "Benjamin Bolte"]); + assert_eq!(e0.categories, vec!["cs.CL", "cs.SD"]); + assert_eq!(e0.primary_category.as_deref(), Some("cs.CL")); + assert_eq!(e0.published.as_deref(), Some("2021-06-14T17:46:43Z")); + assert_eq!( + e0.pdf_link.as_deref(), + Some("http://arxiv.org/pdf/2106.07447v1.pdf") + ); + + let e1 = &entries[1]; + assert_eq!(e1.record_id, "1706.03762v5"); + assert_eq!(e1.authors, vec!["Ashish Vaswani", "Noam Shazeer"]); + } + + #[test] + fn into_discovery_record_emits_required_fields() { + let entries = parse_arxiv_feed(SAMPLE_FEED.as_bytes()).expect("parse"); + let rec = into_discovery_record(entries.into_iter().next().unwrap()); + assert_eq!(rec.upstream_id, "arxiv"); + assert_eq!(rec.record_id, "2106.07447v1"); + // §6.4 conventions. + assert_eq!( + rec.fields.get("fileType").map(String::as_str), + Some("document") + ); + assert_eq!( + rec.fields.get("contentKind").map(String::as_str), + Some("paper") + ); + assert_eq!(rec.fields.get("format").map(String::as_str), Some("pdf")); + assert_eq!( + rec.fields.get("sourceUrl").map(String::as_str), + Some("https://arxiv.org/abs/2106.07447v1") + ); + assert_eq!( + rec.fields.get("fileName").map(String::as_str), + Some("arxiv-2106.07447v1.pdf") + ); + // Canonical `id` field — required by §6.4 for + // typed search filters. + assert_eq!( + rec.fields.get("arxivid").map(String::as_str), + Some("2106.07447v1") + ); + // Year derived from published timestamp. + assert_eq!(rec.fields.get("year").map(String::as_str), Some("2021")); + // Author list flattened. + assert_eq!( + rec.fields.get("author").map(String::as_str), + Some("Wei-Ning Hsu, Benjamin Bolte") + ); + // Categories flattened. + assert_eq!( + rec.fields.get("categories").map(String::as_str), + Some("cs.CL, cs.SD") + ); + } + + #[test] + fn strip_abs_prefix_accepts_both_schemes() { + assert_eq!( + strip_abs_prefix("http://arxiv.org/abs/2106.07447v1"), + "2106.07447v1" + ); + assert_eq!( + strip_abs_prefix("https://arxiv.org/abs/2106.07447"), + "2106.07447" + ); + assert_eq!(strip_abs_prefix("https://export.arxiv.org/abs/foo"), "foo"); + assert_eq!(strip_abs_prefix("not-an-arxiv-url"), "not-an-arxiv-url"); + } + + #[test] + fn rewrite_host_swaps_authority() { + assert_eq!( + rewrite_host( + "http://arxiv.org/pdf/2106.07447v1.pdf", + "https://export.arxiv.org" + ), + Some("https://export.arxiv.org/pdf/2106.07447v1.pdf".to_string()) + ); + assert_eq!(rewrite_host("not-a-url", "https://example.com"), None); + } + + #[test] + fn normalize_whitespace_collapses_runs() { + assert_eq!( + normalize_whitespace(" Hello\n world\t! "), + "Hello world !" + ); + } + + // -- HTTP integration tests against a wiremock server ---------------- + + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn configured_plugin_against(server: &MockServer) -> (ArxivPlugin, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("tempdir"); + let mut plugin = ArxivPlugin::with_base_url(server.uri()); + plugin.configure(dir.path()).expect("configure"); + (plugin, dir) + } + + #[tokio::test] + async fn handle_query_maps_atom_response_to_discovery_records() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/query")) + .and(query_param("search_query", "all:test")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(SAMPLE_FEED) + .insert_header("content-type", "application/atom+xml"), + ) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("test"), 10) + .await + .expect("handle_query"); + assert_eq!(records.len(), 2); + assert_eq!(records[0].record_id, "2106.07447v1"); + assert_eq!( + records[0].fields.get("arxivid").map(String::as_str), + Some("2106.07447v1") + ); + } + + #[tokio::test] + async fn handle_query_truncates_to_max_results() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/query")) + .respond_with(ResponseTemplate::new(200).set_body_string(SAMPLE_FEED)) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text("test"), 1) + .await + .expect("handle_query"); + assert_eq!(records.len(), 1); + } + + #[tokio::test] + async fn compute_outcomes_fetches_pdf_and_caches() { + let server = MockServer::start().await; + // Single-entry id_list response — same shape as a search feed + // but with one entry. The plugin's `fetch_entry` calls this. + let single = r#" + + + http://arxiv.org/abs/2106.07447v1 + HuBERT + 2021-06-14T17:46:43Z + Author One + + + + + +"#; + Mock::given(method("GET")) + .and(path("/api/query")) + .and(query_param("id_list", "2106.07447v1")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(single) + .insert_header("content-type", "application/atom+xml"), + ) + .mount(&server) + .await; + // PDF response — bytes get hashed; the test asserts the + // resulting CID is the canonical sha256/multihash of these bytes. + let pdf_bytes = b"%PDF-1.4 fake pdf body for hashing\n".to_vec(); + let expected_cid = meta_feeder_sdk::hash::compute_ipfs_cid(&pdf_bytes); + Mock::given(method("GET")) + .and(path("/pdf/2106.07447v1.pdf")) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(pdf_bytes.clone()) + .insert_header("content-type", "application/pdf"), + ) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let outcomes = plugin + .compute_outcomes("2106.07447v1") + .await + .expect("compute"); + assert_eq!(outcomes.len(), 1, "arxiv is single-outcome"); + let outcome = &outcomes[0]; + assert_eq!(outcome.hash.as_str(), expected_cid); + assert_eq!(outcome.hash_kind, HashKind::Sha2_256); + // Auto-store inputs: bytes + record + file_extension. + assert_eq!(outcome.bytes.as_deref(), Some(pdf_bytes.as_slice())); + assert_eq!(outcome.file_extension.as_deref(), Some("pdf")); + let rec = outcome + .record + .as_ref() + .expect("record present on fresh compute"); + assert_eq!(rec.record_id, "2106.07447v1"); + assert_eq!( + rec.fields.get("primaryCategory").map(String::as_str), + Some("cs.CL") + ); + } + + #[tokio::test] + async fn compute_outcomes_cache_hit_skips_http() { + let server = MockServer::start().await; + let (plugin, _dir) = configured_plugin_against(&server); + // Seed the cache directly so the plugin won't issue any HTTP. + plugin + .cache + .as_ref() + .unwrap() + .put_midhash("2106.07447v1", "bafyCACHED") + .unwrap(); + let outcomes = plugin + .compute_outcomes("2106.07447v1") + .await + .expect("cache hit"); + assert_eq!(outcomes.len(), 1); + let outcome = &outcomes[0]; + assert_eq!(outcome.hash.as_str(), "bafyCACHED"); + // Cache hits return sparse outcomes — the auto-store side + // effect has already happened on the original miss. + assert!(outcome.bytes.is_none()); + assert!(outcome.record.is_none()); + } + + /// Partial-success contract guard (§1.5): a hard upstream failure + /// must still return `Err`, NOT `Ok(vec![])`. `Ok(vec![])` means + /// "tried, confirmed empty" — distinct semantics. Drift here would + /// silently turn errors into empty bundles. + #[tokio::test] + async fn upstream_404_maps_to_not_found() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/api/query")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let err = plugin + .compute_outcomes("does-not-exist") + .await + .expect_err("expected NotFound (Err, not Ok(vec![]))"); + assert!(matches!(err, GatewayError::NotFound), "got {err:?}"); + } +} diff --git a/feeder-plugin/arxiv-feeder/src/lib.rs b/feeder-plugin/arxiv-feeder/src/lib.rs new file mode 100644 index 0000000..2aa6fc1 --- /dev/null +++ b/feeder-plugin/arxiv-feeder/src/lib.rs @@ -0,0 +1,4 @@ +//! `arxiv-feeder` library surface. The binary (`main.rs`) wraps the plugin in +//! [`meta_feeder_sdk::serve_feeders`]. + +pub mod arxiv; diff --git a/feeder-plugin/arxiv-feeder/src/main.rs b/feeder-plugin/arxiv-feeder/src/main.rs new file mode 100644 index 0000000..ba7feb3 --- /dev/null +++ b/feeder-plugin/arxiv-feeder/src/main.rs @@ -0,0 +1,30 @@ +//! `arxiv-feeder` — a single-upstream feeder sidecar. +//! +//! Upstream: arXiv (export.arxiv.org) +//! +//! Env: +//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`) +//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`) +//! - `RUST_LOG` — tracing filter (default `info`) + +use std::net::SocketAddr; + +use meta_feeder_sdk::{serve_feeders, FeederPlugin}; +use arxiv_feeder::arxiv::ArxivPlugin; +use tracing_subscriber::EnvFilter; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + + let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN") + .unwrap_or_else(|_| "0.0.0.0:8080".to_string()) + .parse()?; + let state_dir = + std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string()); + + let plugins: Vec> = vec![Box::new(ArxivPlugin::new())]; + serve_feeders(plugins, state_dir, listen).await +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..1c0730c --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.89" +components = ["rustfmt", "clippy"]