From 92b2f8aa0df731a2d6dae94bdc697d042e307af1 Mon Sep 17 00:00:00 2001 From: Manielle <75130972+Manielle@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:34:00 +0200 Subject: [PATCH] browse the newest papers when a query has no keywords A query with only filters (a consumer's `domain:science contentKind:paper` row) reached the plugin with no free text, and the plugin substituted `all:*`. arXiv answers that with HTTP 500, so every such query returned nothing, however many papers existed. An empty query (or the SDK's `*` placeholder) now browses the major subject archives (`cat:.*`, OR-ed) newest first (`sortBy=submittedDate`). Keyword queries are unchanged: same `all:` field, same URL, relevance order. The URL is built by a pure function, so both shapes are tested without the network. Every form was verified against the live API: `all:*` -> 500, the browse query -> 200 with 60 entries out of ~2.6M papers. No `licence` is written, on purpose: arXiv declares none per paper, its default licence does not permit redistribution, and the registry says absence means "unknown", never "unrestricted". --- feeder-plugin/arxiv-feeder/src/arxiv.rs | 109 ++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 9 deletions(-) diff --git a/feeder-plugin/arxiv-feeder/src/arxiv.rs b/feeder-plugin/arxiv-feeder/src/arxiv.rs index bd838a5..464b367 100644 --- a/feeder-plugin/arxiv-feeder/src/arxiv.rs +++ b/feeder-plugin/arxiv-feeder/src/arxiv.rs @@ -1,6 +1,8 @@ //! arXiv bridge via the public [arXiv API](https://info.arxiv.org/help/api/index.html). //! //! - Search: `GET /api/query?search_query=all:&max_results=N` → Atom feed. +//! - Browse (a query with no keywords): the newest papers across the major +//! subject archives, `sortBy=submittedDate` — see [`BROWSE_QUERY`]. //! - Fetch one: `GET /api/query?id_list=` → Atom feed (single entry). //! - Canonical file format for midhash: the entry's PDF link //! (``). arXiv always provides one; fall back @@ -48,6 +50,40 @@ const HTTP_TIMEOUT_SECS: u64 = 30; /// Polite identification per the arXiv API guidelines. const USER_AGENT: &str = concat!("meta-share/", env!("CARGO_PKG_VERSION"), " (gateway:arxiv)"); +/// What a query with **no keywords** browses: the newest papers across the +/// major subject archives. +/// +/// ⚠ This exists because the obvious default does not work. A board row such as +/// meta-read's `domain:science contentKind:paper` carries only filters, so the +/// feeder receives no free text; it used to substitute `all:*`, and arXiv's API +/// answers that with **HTTP 500** (verified against the live API, as is every +/// form below). The plugin then returned nothing for every such query, so a +/// consumer's "papers" row stayed empty no matter how many papers existed. +/// +/// `cat:.*` wildcards are accepted and OR-able; sorting by submission +/// date makes the result "what is new" instead of an arbitrary slice. Written +/// pre-encoded (`+` = space) because it is a constant, not user input. +const BROWSE_QUERY: &str = "cat:cs.*+OR+cat:math.*+OR+cat:physics.*+OR+cat:stat.*+OR+cat:q-bio.*+OR+cat:astro-ph.*+OR+cat:cond-mat.*+OR+cat:econ.*+OR+cat:eess.*+OR+cat:q-fin.*"; + +/// The Atom-search URL for `query`. +/// +/// Keywords search the `all:` field exactly as before. An empty query, or the +/// SDK's `*` placeholder for one, browses [`BROWSE_QUERY`] newest first. +fn search_url(base: &str, query: &str, max_results: usize) -> String { + let base = base.trim_end_matches('/'); + let q = query.trim(); + if q.is_empty() || q == "*" { + format!( + "{base}/api/query?search_query={BROWSE_QUERY}&sortBy=submittedDate&sortOrder=descending&start=0&max_results={max_results}" + ) + } else { + format!( + "{base}/api/query?search_query=all:{}&start=0&max_results={max_results}", + common::urlencode(q) + ) + } +} + /// arXiv gateway plugin. Same structural shape as gutenberg: cheap to /// construct; `configure()` opens the per-plugin redb cache. The cache /// stores `(record_id → midhash)` so we don't re-download a multi-MB @@ -86,16 +122,12 @@ impl ArxivPlugin { common::require_cache(self.cache.as_ref(), "arxiv") } - /// Search the arXiv Atom API. `query` is passed verbatim as the - /// `all:` field, so callers get the full-text behaviour arXiv users - /// expect (free-text + boolean syntax both work upstream). + /// Search the arXiv Atom API. A non-empty `query` is passed verbatim as the + /// `all:` field, so callers get the full-text behaviour arXiv users expect + /// (free-text + boolean syntax both work upstream); an empty one browses + /// the newest papers (see [`search_url`]). async fn search_atom(&self, query: &str, max_results: usize) -> Result, GatewayError> { - let url = format!( - "{}/api/query?search_query=all:{}&start=0&max_results={}", - self.base_url.trim_end_matches('/'), - common::urlencode(query), - max_results - ); + let url = search_url(&self.base_url, query, max_results); common::fetch_bytes(&self.http, &url) .await .map(|b| b.to_vec()) @@ -848,4 +880,63 @@ mod tests { .expect_err("expected NotFound (Err, not Ok(vec![]))"); assert!(matches!(err, GatewayError::NotFound), "got {err:?}"); } + + // -- keyword-less queries browse instead of asking arXiv for `all:*` ------- + + #[test] + fn keywords_search_the_all_field_unchanged() { + let url = search_url("https://export.arxiv.org/", "transformer attention", 20); + assert_eq!( + url, + "https://export.arxiv.org/api/query?search_query=all:transformer+attention&start=0&max_results=20" + ); + assert!(!url.contains("sortBy"), "a keyword search keeps arXiv's relevance order"); + } + + #[test] + fn an_empty_query_browses_newest_first() { + for q in ["", " ", "*", " * "] { + let url = search_url("https://export.arxiv.org", q, 30); + assert!(url.contains("cat:cs.*"), "{q:?} -> {url}"); + assert!(url.contains("sortBy=submittedDate&sortOrder=descending"), "{q:?} -> {url}"); + assert!(url.ends_with("max_results=30"), "{q:?} -> {url}"); + assert!(!url.contains("all:"), "the all:* form returns HTTP 500: {q:?} -> {url}"); + } + } + + #[tokio::test] + async fn a_keywordless_query_returns_papers_rather_than_nothing() { + use wiremock::matchers::query_param_contains; + let server = MockServer::start().await; + // The form that used to be sent. arXiv answers it with a 500; if the + // plugin ever goes back to it this mock is hit and the test fails. + Mock::given(method("GET")) + .and(path("/api/query")) + .and(query_param("search_query", "all:*")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/api/query")) + .and(query_param("sortBy", "submittedDate")) + .and(query_param_contains("search_query", "cat:cs.*")) + .respond_with( + ResponseTemplate::new(200) + .set_body_string(SAMPLE_FEED) + .insert_header("content-type", "application/atom+xml"), + ) + .mount(&server) + .await; + let (plugin, _dir) = configured_plugin_against(&server); + let records = plugin + .handle_query(&GatewayQuery::from_free_text(""), 10) + .await + .expect("a keyword-less query must succeed"); + assert_eq!(records.len(), 2); + assert_eq!( + records[0].fields.get("contentKind").map(String::as_str), + Some("paper") + ); + } }