diff --git a/feeder-plugin/arxiv-feeder/src/arxiv.rs b/feeder-plugin/arxiv-feeder/src/arxiv.rs index 464b367..9d114e2 100644 --- a/feeder-plugin/arxiv-feeder/src/arxiv.rs +++ b/feeder-plugin/arxiv-feeder/src/arxiv.rs @@ -551,6 +551,29 @@ fn into_discovery_record(entry: ArxivEntry) -> DiscoveryRecord { fields.insert("fileName".to_string(), format!("arxiv-{record_id}.pdf")); fields.insert(upstream_id_field("arxiv"), record_id.clone()); fields.insert("format".to_string(), "pdf".to_string()); + // ⚠ A record with no CID is invisible to everything downstream of the + // gateway. The gateway persists only search hits that carry at least one + // CID ("search-persist: record carries no CID; not persisted"), but it still + // stamps the query as covered for an hour (META_GATEWAY_SEARCH_COVERAGE_TTL_ + // SECS) and, inside that window, skips this feeder and answers from what it + // persisted. So the first query returned papers, and every identical one for + // the next hour returned none. A client also needs a CID to open a paper at + // all. Measured on the live box: 40 hits, then zero hits in 1.5 s. + // + // The fleet convention (Gutenberg, internetarchive, jamendo): a `0x1006` + // url-locator over the file's URL. It is a pure function of the URL, so it + // costs no I/O, and it doubles as the byte path - meta-share redeems the + // locator on demand. Built from the canonical, *versioned* PDF URL rather + // than the feed's own link, which arXiv emits as `http://` without `.pdf`: + // v2 of a paper is a different file and must be a different CID. + // + // ⚠ Exactly ONE `cids/` member: the store key is the lexicographic `.min()` + // of them, so a second one would make it arbitrary. + if let Some(cid) = + meta_feeder_sdk::hash::compute_url_cid(&format!("https://arxiv.org/pdf/{record_id}.pdf")) + { + fields.insert(format!("cids/{cid}"), "true".to_string()); + } if !entry.authors.is_empty() { fields.insert("author".to_string(), entry.authors.join(", ")); } @@ -881,6 +904,35 @@ mod tests { assert!(matches!(err, GatewayError::NotFound), "got {err:?}"); } + // -- a record the gateway can persist ---------------------------------------- + + #[test] + fn a_record_carries_exactly_one_url_locator_cid_over_the_versioned_pdf() { + let entries = parse_arxiv_feed(SAMPLE_FEED.as_bytes()).expect("parse"); + for e in entries { + let rec = into_discovery_record(e); + let cids: Vec<&String> = rec.fields.keys().filter(|k| k.starts_with("cids/")).collect(); + assert_eq!(cids.len(), 1, "exactly one cids/ member: {cids:?}"); + let cid = cids[0].trim_start_matches("cids/"); + // The locator is the URL itself, recoverable with no I/O - which is + // what lets meta-share redeem it as the byte path. + assert_eq!( + meta_feeder_sdk::hash::decode_url_cid(cid).as_deref(), + Some(format!("https://arxiv.org/pdf/{}.pdf", rec.record_id).as_str()) + ); + } + } + + #[test] + fn two_versions_of_a_paper_get_two_cids() { + let mut feed = SAMPLE_FEED.replace("2106.07447v1", "2106.07447v2"); + let v2 = into_discovery_record(parse_arxiv_feed(feed.as_bytes()).unwrap().remove(0)); + feed = SAMPLE_FEED.to_string(); + let v1 = into_discovery_record(parse_arxiv_feed(feed.as_bytes()).unwrap().remove(0)); + let key = |r: &DiscoveryRecord| r.fields.keys().find(|k| k.starts_with("cids/")).cloned(); + assert_ne!(key(&v1), key(&v2), "v2 is a different file, so a different CID"); + } + // -- keyword-less queries browse instead of asking arXiv for `all:*` ------- #[test]