browse the newest papers when a query has no keywords

A query with only filters (a consumer's `domain:science contentKind:paper` row) reached the plugin with no free text, and the plugin substituted `all:*`. arXiv answers that with HTTP 500, so every such query returned nothing, however many papers existed. An empty query (or the SDK's `*` placeholder) now browses the major subject archives (`cat:<archive>.*`, OR-ed) newest first (`sortBy=submittedDate`). Keyword queries are unchanged: same `all:` field, same URL, relevance order. The URL is built by a pure function, so both shapes are tested without the network. Every form was verified against the live API: `all:*` -> 500, the browse query -> 200 with 60 entries out of ~2.6M papers. No `licence` is written, on purpose: arXiv declares none per paper, its default licence does not permit redistribution, and the registry says absence means "unknown", never "unrestricted".
2026-10-11 16:34:00 +02:00
parent 1d4eee7042
commit 92b2f8aa0d
1 changed files with 100 additions and 9 deletions
@@ -1,6 +1,8 @@
 //! arXiv bridge via the public [arXiv API](https://info.arxiv.org/help/api/index.html).
 //!
 //! - Search: `GET /api/query?search_query=all:<q>&max_results=N` → Atom feed.
+//! - Browse (a query with no keywords): the newest papers across the major
+//!   subject archives, `sortBy=submittedDate` — see [`BROWSE_QUERY`].
 //! - Fetch one: `GET /api/query?id_list=<record_id>` → Atom feed (single entry).
 //! - Canonical file format for midhash: the entry's PDF link
 //!   (`<link title="pdf" href="…">`). arXiv always provides one; fall back
@@ -48,6 +50,40 @@ const HTTP_TIMEOUT_SECS: u64 = 30;
 /// Polite identification per the arXiv API guidelines.
 const USER_AGENT: &str = concat!("meta-share/", env!("CARGO_PKG_VERSION"), " (gateway:arxiv)");

+/// What a query with **no keywords** browses: the newest papers across the
+/// major subject archives.
+///
+/// ⚠ This exists because the obvious default does not work. A board row such as
+/// meta-read's `domain:science contentKind:paper` carries only filters, so the
+/// feeder receives no free text; it used to substitute `all:*`, and arXiv's API
+/// answers that with **HTTP 500** (verified against the live API, as is every
+/// form below). The plugin then returned nothing for every such query, so a
+/// consumer's "papers" row stayed empty no matter how many papers existed.
+///
+/// `cat:<archive>.*` wildcards are accepted and OR-able; sorting by submission
+/// date makes the result "what is new" instead of an arbitrary slice. Written
+/// pre-encoded (`+` = space) because it is a constant, not user input.
+const BROWSE_QUERY: &str = "cat:cs.*+OR+cat:math.*+OR+cat:physics.*+OR+cat:stat.*+OR+cat:q-bio.*+OR+cat:astro-ph.*+OR+cat:cond-mat.*+OR+cat:econ.*+OR+cat:eess.*+OR+cat:q-fin.*";
+
+/// The Atom-search URL for `query`.
+///
+/// Keywords search the `all:` field exactly as before. An empty query, or the
+/// SDK's `*` placeholder for one, browses [`BROWSE_QUERY`] newest first.
+fn search_url(base: &str, query: &str, max_results: usize) -> String {
+    let base = base.trim_end_matches('/');
+    let q = query.trim();
+    if q.is_empty() || q == "*" {
+        format!(
+            "{base}/api/query?search_query={BROWSE_QUERY}&sortBy=submittedDate&sortOrder=descending&start=0&max_results={max_results}"
+        )
+    } else {
+        format!(
+            "{base}/api/query?search_query=all:{}&start=0&max_results={max_results}",
+            common::urlencode(q)
+        )
+    }
+}
+
 /// arXiv gateway plugin. Same structural shape as gutenberg: cheap to
 /// construct; `configure()` opens the per-plugin redb cache. The cache
 /// stores `(record_id → midhash)` so we don't re-download a multi-MB
@@ -86,16 +122,12 @@ impl ArxivPlugin {
        common::require_cache(self.cache.as_ref(), "arxiv")
    }

-    /// Search the arXiv Atom API. `query` is passed verbatim as the
-    /// `all:` field, so callers get the full-text behaviour arXiv users
-    /// expect (free-text + boolean syntax both work upstream).
+    /// Search the arXiv Atom API. A non-empty `query` is passed verbatim as the
+    /// `all:` field, so callers get the full-text behaviour arXiv users expect
+    /// (free-text + boolean syntax both work upstream); an empty one browses
+    /// the newest papers (see [`search_url`]).
    async fn search_atom(&self, query: &str, max_results: usize) -> Result<Vec<u8>, GatewayError> {
-        let url = format!(
-            "{}/api/query?search_query=all:{}&start=0&max_results={}",
-            self.base_url.trim_end_matches('/'),
-            common::urlencode(query),
-            max_results
-        );
+        let url = search_url(&self.base_url, query, max_results);
        common::fetch_bytes(&self.http, &url)
            .await
            .map(|b| b.to_vec())
@@ -848,4 +880,63 @@ mod tests {
            .expect_err("expected NotFound (Err, not Ok(vec![]))");
        assert!(matches!(err, GatewayError::NotFound), "got {err:?}");
    }
+
+    // -- keyword-less queries browse instead of asking arXiv for `all:*` -------
+
+    #[test]
+    fn keywords_search_the_all_field_unchanged() {
+        let url = search_url("https://export.arxiv.org/", "transformer attention", 20);
+        assert_eq!(
+            url,
+            "https://export.arxiv.org/api/query?search_query=all:transformer+attention&start=0&max_results=20"
+        );
+        assert!(!url.contains("sortBy"), "a keyword search keeps arXiv's relevance order");
+    }
+
+    #[test]
+    fn an_empty_query_browses_newest_first() {
+        for q in ["", "   ", "*", " * "] {
+            let url = search_url("https://export.arxiv.org", q, 30);
+            assert!(url.contains("cat:cs.*"), "{q:?} -> {url}");
+            assert!(url.contains("sortBy=submittedDate&sortOrder=descending"), "{q:?} -> {url}");
+            assert!(url.ends_with("max_results=30"), "{q:?} -> {url}");
+            assert!(!url.contains("all:"), "the all:* form returns HTTP 500: {q:?} -> {url}");
+        }
+    }
+
+    #[tokio::test]
+    async fn a_keywordless_query_returns_papers_rather_than_nothing() {
+        use wiremock::matchers::query_param_contains;
+        let server = MockServer::start().await;
+        // The form that used to be sent. arXiv answers it with a 500; if the
+        // plugin ever goes back to it this mock is hit and the test fails.
+        Mock::given(method("GET"))
+            .and(path("/api/query"))
+            .and(query_param("search_query", "all:*"))
+            .respond_with(ResponseTemplate::new(500))
+            .expect(0)
+            .mount(&server)
+            .await;
+        Mock::given(method("GET"))
+            .and(path("/api/query"))
+            .and(query_param("sortBy", "submittedDate"))
+            .and(query_param_contains("search_query", "cat:cs.*"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_string(SAMPLE_FEED)
+                    .insert_header("content-type", "application/atom+xml"),
+            )
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let records = plugin
+            .handle_query(&GatewayQuery::from_free_text(""), 10)
+            .await
+            .expect("a keyword-less query must succeed");
+        assert_eq!(records.len(), 2);
+        assert_eq!(
+            records[0].fields.get("contentKind").map(String::as_str),
+            Some("paper")
+        );
+    }
 }