give every record a url-locator CID so the gateway can persist it

A record with no CID cannot be persisted by the gateway ("search-persist: record
carries no CID; not persisted"), yet the gateway still stamps the query as
covered for an hour (META_GATEWAY_SEARCH_COVERAGE_TTL_SECS) and, inside that
window, skips this feeder and answers from what it persisted. So the first query
returned papers and every identical one for the next hour returned none.
Measured on the live box: 40 hits, then 0 hits in 1.5 s with no fan-out.

Each record now carries exactly one `cids/` member: a 0x1006 url-locator over the
canonical, versioned PDF URL (https://arxiv.org/pdf/<id>.pdf). This is the fleet
convention (Gutenberg, internetarchive, jamendo): it is a pure function of the
URL, so it costs no I/O, and it doubles as the byte path that meta-share
redeems. The versioned id means v2 of a paper is a different CID. Built from the
canonical URL rather than the feed's link, which arXiv emits as http:// without
.pdf.

Verified live after deploying: 40 hits each carrying a CID; the same query again
inside its coverage window returns 40 (was 0); meta-share serves the locator
with 206 Partial Content and the bytes are a real PDF (%PDF-1.7, 1,182,901 B).

Cargo.lock

@@ -762,8 +762,8 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
 
 [[package]]
 name = "meta-feeder-sdk"
-version = "0.1.0"
+version = "1.7.0"
-source = "git+https://github.com/worph/meta-feeder-sdk?rev=d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4#d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4"
+source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -772,6 +772,7 @@ dependencies = [
  "bytes",
  "futures",
  "hyper",
+ "libc",
  "redb",
  "regex",
  "reqwest",

feeder-plugin/arxiv-feeder/Cargo.toml

@@ -20,7 +20,7 @@ path = "src/main.rs"
 [dependencies]
 # The SDK is a dependency, not a vendored copy. Pinned by rev until the first
 # tag; see docs/project-architecture/feeder-architecture.md §8.
-meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "d2c6ef4c1887be9ae9f92ebedc39a810aa57c7e4" }
+meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
 
 tokio = { version = "1.40", features = ["full"] }
 axum = { version = "0.7", features = ["json"] }

feeder-plugin/arxiv-feeder/src/arxiv.rs

@@ -1,6 +1,8 @@
 //! arXiv bridge via the public [arXiv API](https://info.arxiv.org/help/api/index.html).
 //!
 //! - Search: `GET /api/query?search_query=all:<q>&max_results=N` → Atom feed.
+//! - Browse (a query with no keywords): the newest papers across the major
+//!   subject archives, `sortBy=submittedDate` — see [`BROWSE_QUERY`].
 //! - Fetch one: `GET /api/query?id_list=<record_id>` → Atom feed (single entry).
 //! - Canonical file format for midhash: the entry's PDF link
 //!   (`<link title="pdf" href="…">`). arXiv always provides one; fall back
@@ -48,6 +50,40 @@ const HTTP_TIMEOUT_SECS: u64 = 30;
 /// Polite identification per the arXiv API guidelines.
 const USER_AGENT: &str = concat!("meta-share/", env!("CARGO_PKG_VERSION"), " (gateway:arxiv)");
 
+/// What a query with **no keywords** browses: the newest papers across the
+/// major subject archives.
+///
+/// ⚠ This exists because the obvious default does not work. A board row such as
+/// meta-read's `domain:science contentKind:paper` carries only filters, so the
+/// feeder receives no free text; it used to substitute `all:*`, and arXiv's API
+/// answers that with **HTTP 500** (verified against the live API, as is every
+/// form below). The plugin then returned nothing for every such query, so a
+/// consumer's "papers" row stayed empty no matter how many papers existed.
+///
+/// `cat:<archive>.*` wildcards are accepted and OR-able; sorting by submission
+/// date makes the result "what is new" instead of an arbitrary slice. Written
+/// pre-encoded (`+` = space) because it is a constant, not user input.
+const BROWSE_QUERY: &str = "cat:cs.*+OR+cat:math.*+OR+cat:physics.*+OR+cat:stat.*+OR+cat:q-bio.*+OR+cat:astro-ph.*+OR+cat:cond-mat.*+OR+cat:econ.*+OR+cat:eess.*+OR+cat:q-fin.*";
+
+/// The Atom-search URL for `query`.
+///
+/// Keywords search the `all:` field exactly as before. An empty query, or the
+/// SDK's `*` placeholder for one, browses [`BROWSE_QUERY`] newest first.
+fn search_url(base: &str, query: &str, max_results: usize) -> String {
+    let base = base.trim_end_matches('/');
+    let q = query.trim();
+    if q.is_empty() || q == "*" {
+        format!(
+            "{base}/api/query?search_query={BROWSE_QUERY}&sortBy=submittedDate&sortOrder=descending&start=0&max_results={max_results}"
+        )
+    } else {
+        format!(
+            "{base}/api/query?search_query=all:{}&start=0&max_results={max_results}",
+            common::urlencode(q)
+        )
+    }
+}
+
 /// arXiv gateway plugin. Same structural shape as gutenberg: cheap to
 /// construct; `configure()` opens the per-plugin redb cache. The cache
 /// stores `(record_id → midhash)` so we don't re-download a multi-MB
@@ -86,16 +122,12 @@ impl ArxivPlugin {
         common::require_cache(self.cache.as_ref(), "arxiv")
     }
 
-    /// Search the arXiv Atom API. `query` is passed verbatim as the
+    /// Search the arXiv Atom API. A non-empty `query` is passed verbatim as the
-    /// `all:` field, so callers get the full-text behaviour arXiv users
+    /// `all:` field, so callers get the full-text behaviour arXiv users expect
-    /// expect (free-text + boolean syntax both work upstream).
+    /// (free-text + boolean syntax both work upstream); an empty one browses
+    /// the newest papers (see [`search_url`]).
     async fn search_atom(&self, query: &str, max_results: usize) -> Result<Vec<u8>, GatewayError> {
-        let url = format!(
+        let url = search_url(&self.base_url, query, max_results);
-            "{}/api/query?search_query=all:{}&start=0&max_results={}",
-            self.base_url.trim_end_matches('/'),
-            common::urlencode(query),
-            max_results
-        );
         common::fetch_bytes(&self.http, &url)
             .await
             .map(|b| b.to_vec())
@@ -519,6 +551,29 @@ fn into_discovery_record(entry: ArxivEntry) -> DiscoveryRecord {
     fields.insert("fileName".to_string(), format!("arxiv-{record_id}.pdf"));
     fields.insert(upstream_id_field("arxiv"), record_id.clone());
     fields.insert("format".to_string(), "pdf".to_string());
+    // ⚠ A record with no CID is invisible to everything downstream of the
+    // gateway. The gateway persists only search hits that carry at least one
+    // CID ("search-persist: record carries no CID; not persisted"), but it still
+    // stamps the query as covered for an hour (META_GATEWAY_SEARCH_COVERAGE_TTL_
+    // SECS) and, inside that window, skips this feeder and answers from what it
+    // persisted. So the first query returned papers, and every identical one for
+    // the next hour returned none. A client also needs a CID to open a paper at
+    // all. Measured on the live box: 40 hits, then zero hits in 1.5 s.
+    //
+    // The fleet convention (Gutenberg, internetarchive, jamendo): a `0x1006`
+    // url-locator over the file's URL. It is a pure function of the URL, so it
+    // costs no I/O, and it doubles as the byte path - meta-share redeems the
+    // locator on demand. Built from the canonical, *versioned* PDF URL rather
+    // than the feed's own link, which arXiv emits as `http://` without `.pdf`:
+    // v2 of a paper is a different file and must be a different CID.
+    //
+    // ⚠ Exactly ONE `cids/` member: the store key is the lexicographic `.min()`
+    // of them, so a second one would make it arbitrary.
+    if let Some(cid) =
+        meta_feeder_sdk::hash::compute_url_cid(&format!("https://arxiv.org/pdf/{record_id}.pdf"))
+    {
+        fields.insert(format!("cids/{cid}"), "true".to_string());
+    }
     if !entry.authors.is_empty() {
         fields.insert("author".to_string(), entry.authors.join(", "));
     }
@@ -848,4 +903,92 @@ mod tests {
             .expect_err("expected NotFound (Err, not Ok(vec![]))");
         assert!(matches!(err, GatewayError::NotFound), "got {err:?}");
     }
+
+    // -- a record the gateway can persist ----------------------------------------
+
+    #[test]
+    fn a_record_carries_exactly_one_url_locator_cid_over_the_versioned_pdf() {
+        let entries = parse_arxiv_feed(SAMPLE_FEED.as_bytes()).expect("parse");
+        for e in entries {
+            let rec = into_discovery_record(e);
+            let cids: Vec<&String> = rec.fields.keys().filter(|k| k.starts_with("cids/")).collect();
+            assert_eq!(cids.len(), 1, "exactly one cids/ member: {cids:?}");
+            let cid = cids[0].trim_start_matches("cids/");
+            // The locator is the URL itself, recoverable with no I/O - which is
+            // what lets meta-share redeem it as the byte path.
+            assert_eq!(
+                meta_feeder_sdk::hash::decode_url_cid(cid).as_deref(),
+                Some(format!("https://arxiv.org/pdf/{}.pdf", rec.record_id).as_str())
+            );
+        }
+    }
+
+    #[test]
+    fn two_versions_of_a_paper_get_two_cids() {
+        let mut feed = SAMPLE_FEED.replace("2106.07447v1", "2106.07447v2");
+        let v2 = into_discovery_record(parse_arxiv_feed(feed.as_bytes()).unwrap().remove(0));
+        feed = SAMPLE_FEED.to_string();
+        let v1 = into_discovery_record(parse_arxiv_feed(feed.as_bytes()).unwrap().remove(0));
+        let key = |r: &DiscoveryRecord| r.fields.keys().find(|k| k.starts_with("cids/")).cloned();
+        assert_ne!(key(&v1), key(&v2), "v2 is a different file, so a different CID");
+    }
+
+    // -- keyword-less queries browse instead of asking arXiv for `all:*` -------
+
+    #[test]
+    fn keywords_search_the_all_field_unchanged() {
+        let url = search_url("https://export.arxiv.org/", "transformer attention", 20);
+        assert_eq!(
+            url,
+            "https://export.arxiv.org/api/query?search_query=all:transformer+attention&start=0&max_results=20"
+        );
+        assert!(!url.contains("sortBy"), "a keyword search keeps arXiv's relevance order");
+    }
+
+    #[test]
+    fn an_empty_query_browses_newest_first() {
+        for q in ["", "   ", "*", " * "] {
+            let url = search_url("https://export.arxiv.org", q, 30);
+            assert!(url.contains("cat:cs.*"), "{q:?} -> {url}");
+            assert!(url.contains("sortBy=submittedDate&sortOrder=descending"), "{q:?} -> {url}");
+            assert!(url.ends_with("max_results=30"), "{q:?} -> {url}");
+            assert!(!url.contains("all:"), "the all:* form returns HTTP 500: {q:?} -> {url}");
+        }
+    }
+
+    #[tokio::test]
+    async fn a_keywordless_query_returns_papers_rather_than_nothing() {
+        use wiremock::matchers::query_param_contains;
+        let server = MockServer::start().await;
+        // The form that used to be sent. arXiv answers it with a 500; if the
+        // plugin ever goes back to it this mock is hit and the test fails.
+        Mock::given(method("GET"))
+            .and(path("/api/query"))
+            .and(query_param("search_query", "all:*"))
+            .respond_with(ResponseTemplate::new(500))
+            .expect(0)
+            .mount(&server)
+            .await;
+        Mock::given(method("GET"))
+            .and(path("/api/query"))
+            .and(query_param("sortBy", "submittedDate"))
+            .and(query_param_contains("search_query", "cat:cs.*"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_string(SAMPLE_FEED)
+                    .insert_header("content-type", "application/atom+xml"),
+            )
+            .mount(&server)
+            .await;
+        let (plugin, _dir) = configured_plugin_against(&server);
+        let records = plugin
+            .handle_query(&GatewayQuery::from_free_text(""), 10)
+            .await
+            .expect("a keyword-less query must succeed");
+        assert_eq!(records.len(), 2);
+        assert_eq!(
+            records[0].fields.get("contentKind").map(String::as_str),
+            Some("paper")
+        );
+    }
 }