From ba3b0a5cb8c21d8a7e561f0734c16cf215292c42 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 31 Aug 2026 21:26:25 +0200 Subject: [PATCH] buffix --- crates/meta-feeder-sdk/src/hash.rs | 33 +++++++++++++ feeder-plugin/book-feeder/src/gutenberg.rs | 54 +++++++++++++++------- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/crates/meta-feeder-sdk/src/hash.rs b/crates/meta-feeder-sdk/src/hash.rs index 2ed0cde..56c0b77 100644 --- a/crates/meta-feeder-sdk/src/hash.rs +++ b/crates/meta-feeder-sdk/src/hash.rs @@ -246,6 +246,39 @@ fn decode_hex_id(id: &str) -> Option> { /// * It ranks `RANK_LOCATOR` (5), so it can never outrank a real digest for /// `canonical_cid` — which is what makes the future "one card, two CIDs" /// merge (a TMDB and a MyAnimeList locator on one meta-core record) safe. +/// The `url` locator codec (`0x1006`). +/// +/// ⚠ Back-ported into this vendored SDK copy because the shared crate has it and +/// this one predates it. Keep it byte-identical to +/// `meta-feeder-sdk::hash::compute_url_cid`: it is how a record gets an address +/// without the feeder ever fetching bytes, and two encoders disagreeing would +/// mint two cids for one URL. +pub const URL_LOCATOR_CODEC: u64 = 0x1006; + +/// Encode an `http(s)` URL as a `0x1006` identity-multihash CIDv1. +/// +/// The digest IS the URL bytes, so this costs no I/O and no lookup — which is +/// what makes it usable at search time, on every record, before anything has +/// been fetched. Resolution happens later, on whichever peer wants the bytes. +/// +/// `None` for anything that is not an absolute http(s) URL: a relative or +/// malformed value would encode fine and then fail unresolvably at read time, +/// which is a far more expensive way to find out. +pub fn compute_url_cid(url: &str) -> Option { + let u = url.trim(); + if !(u.starts_with("http://") || u.starts_with("https://")) { + return None; + } + let digest = u.as_bytes(); + let mut wire = Vec::with_capacity(1 + 3 + 2 + digest.len()); + wire.push(0x01); // CIDv1 + write_pb_varint(URL_LOCATOR_CODEC, &mut wire); + wire.push(0x00); // multihash code: identity + write_pb_varint(digest.len() as u64, &mut wire); + wire.extend_from_slice(digest); + Some(format!("b{}", base32_lower_no_padding(&wire))) +} + pub const CARD_LOCATOR_CODEC: u64 = 0x1007; /// Digest byte ceiling for a card locator, matching meta-share's diff --git a/feeder-plugin/book-feeder/src/gutenberg.rs b/feeder-plugin/book-feeder/src/gutenberg.rs index 4ae7d73..288c3d6 100644 --- a/feeder-plugin/book-feeder/src/gutenberg.rs +++ b/feeder-plugin/book-feeder/src/gutenberg.rs @@ -465,9 +465,19 @@ fn into_discovery_record( if let Some(mt) = media_type { fields.insert("mediaType".into(), mt); } - if let Some(cid) = cover_cid { - fields.insert("poster".into(), cid); - } + // ⚠ Deliberately NOT stamping a self-computed `poster`. + // + // This feeder hashes the cover itself and used to stamp the result. Two + // things were wrong with that. The bytes only ever reached its own redb blob + // cache, so the cid named data no peer on the mesh held — and the cid it + // produced was malformed anyway: 50 characters where a raw sha2-256 CIDv1 is + // 59, so `/api/image/` 504'd even once a real seed existed, because the + // stale field won over the gateway's rewrite. + // + // `poster_url` below is the whole contract: the gateway's SEEDABLE_FIELDS + // path fetches it, hashes the bytes, seeds them into the blockstore and + // writes the resulting content cid back as `poster`. + let _ = cover_cid; // ⚠ `poster` alone is a CID with no bytes behind it anywhere on the mesh. // // The cover is fetched and hashed here, but the bytes only ever land in this @@ -485,22 +495,32 @@ fn into_discovery_record( if let Some(url) = formats.get("image/jpeg") { fields.insert("poster_url".into(), url.clone()); } - if formats.contains_key(EPUB_FORMAT) { + if let Some(epub_url) = formats.get(EPUB_FORMAT) { fields.insert("format".into(), "epub".into()); + // ⚠ Without a `cids/` member this record is SILENTLY DISCARDED. + // + // The gateway keys its store-back on `url_key_for` (meta-gateway + // `dispatch.rs`) and skips any record that has none — while the + // search-coverage gate still marks (gutenberg, query) covered for an + // hour. Together that produced the tier's long-standing behaviour: the + // first search answers from the live fan-out, nothing is ever + // persisted, and every repeat inside the window is served from an empty + // meta-core. Measured: the gateway's meta-core held zero /file records + // ("meta-core poll: nothing new, scanned=0") after hundreds of results, + // which also meant no poster was ever seeded and every cover was blank. + // + // A `0x1006` url-locator over the download URL is the fleet convention — + // meta-feeder-internetarchive and -jamendo both stamp + // `cids/`. It costs no I/O (the cid is a pure + // function of the URL string) and doubles as the byte path: meta-share + // redeems the locator on demand. + // + // ⚠ Exactly ONE `cids/` member. `url_key_for` takes the lexicographic + // `.min()`, so a second cid would make the store key arbitrary. + if let Some(cid) = meta_feeder_sdk::hash::compute_url_cid(epub_url) { + fields.insert(format!("cids/{cid}"), "true".into()); + } } - // ⚠ This record carries NO `cids/` member, and that is why the tier behaves - // as "the first search works, every repeat within the hour returns nothing". - // The gateway keys its store-back on `url_key_for` (meta-gateway - // `dispatch.rs`) and skips any record without one, while the search-coverage - // gate still marks (gutenberg, query) covered — so nothing is ever persisted - // and the gate then serves the empty meta-core. Measured: the gateway's - // meta-core held zero /file records ("meta-core poll: nothing new, - // scanned=0") after hundreds of results. - // - // The fix is the fleet convention — `cids/`, as - // meta-feeder-internetarchive and -jamendo both do — but THIS repo vendors - // an older SDK copy at crates/meta-feeder-sdk that predates - // `compute_url_cid`. Stamping it needs that vendored copy upgraded first. if let Some(c) = copyright { fields.insert("publicDomain".into(), (!c).to_string()); }