Compare commits

..
2 Commits
Author SHA1 Message Date
Worph 80050cd2a8 update vector with card locator
Build and Push Docker Image / build (push) Failing after 13s
2026-07-26 18:49:27 +02:00
Worph 75dda1e7d6 new NZB cid support
Build and Push Docker Image / build (push) Failing after 13s
2026-07-25 21:30:46 +02:00
3 changed files with 76 additions and 12 deletions
+14 -1
View File
@@ -19,9 +19,10 @@
"ladder": { "ladder": {
"40": "dag-pb (IPFS block)", "40": "dag-pb (IPFS block)",
"30": "sha2-256 / sha3-256 / sha3-384 - real content digests", "30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
"25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
"20": "BitTorrent infohashes - swarm interop", "20": "BitTorrent infohashes - swarm interop",
"10": "midhash256 - the record address", "10": "midhash256 - the record address",
"5": "opaque locators (nzb-release, url) - never outrank a real digest", "5": "opaque locators (nzb-release, url, card) - never outrank a real digest",
"0": "weak/unknown digests, unparseable input" "0": "weak/unknown digests, unparseable input"
}, },
"tieBreak": "lexicographically smaller cid wins", "tieBreak": "lexicographically smaller cid wins",
@@ -56,6 +57,12 @@
"rank": 30, "rank": 30,
"note": "REAL meta-hash fixture" "note": "REAL meta-hash fixture"
}, },
{
"name": "nzb-posting",
"cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 25,
"note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
},
{ {
"name": "btih-v2-codec-eq-mh", "name": "btih-v2-codec-eq-mh",
"cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
@@ -103,6 +110,12 @@
"rank": 5, "rank": 5,
"note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release." "note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
}, },
{
"name": "card-locator",
"cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
"rank": 5,
"note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
},
{ {
"name": "sha1", "name": "sha1",
"cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra", "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
+40 -8
View File
@@ -49,16 +49,37 @@ const (
// these — matching on mh would see 0x00 and rank them 0. // these — matching on mh would see 0x00 and rank them 0.
CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
CodeURL = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL CodeURL = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
// CodeCard is the limit case of the family: the only CID with NO bytes
// behind it anywhere, ever. It addresses a *work* (a series, a film) via
// varint(len(source))‖source‖id — ("tmdb","tv:95479") — so `/api/file/{cid}`
// correctly 404s and bitswap never offers it. Ranking it here (rather than
// letting it fall to RankUnknown) is what makes the planned "one card, two
// CIDs" cross-source merge safe: a TMDB and a MyAnimeList locator can sit on
// one record, and a real digest added later still wins the election.
CodeCard = 0x1007 // custom — identity-multihash CID wrapping a card (work) id
// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
// REAL sha2-256 digest (over the normalised, sorted article Message-ID
// set of a Usenet posting we scanned ourselves — never a locator, no
// indexer host embedded), so it would silently fall into RankDigest via
// the mh switch unless matched on the codec FIRST. It is content-derived
// (so it outranks the opaque locators) but addresses no fetchable block —
// the digest is over an id set, not over any block's bytes, so it can
// never be bitswap-fetched under itself — so it must stay BELOW a real
// digest (once the media is materialised and re-seeded, that genuine
// sha2-256 cid has to win the canonical election, not this one).
CodeNzbPosting = 0x1003 // custom — self-scanned Usenet posting identity
) )
// Rank tiers, named so the ladder reads as a ladder. // Rank tiers, named so the ladder reads as a ladder.
const ( const (
RankIPFS = 40 // dag-pb — broadest network reach RankIPFS = 40 // dag-pb — broadest network reach
RankDigest = 30 // sha2-256 / sha3-* — real content digests RankDigest = 30 // sha2-256 / sha3-* — real content digests
RankBtih = 20 // BitTorrent infohashes — swarm interop RankNzbPosting = 25 // content-derived, but the digest addresses no fetchable block
RankMidhash = 10 // the record address — internally meaningful, externally invisible RankBtih = 20 // BitTorrent infohashes — swarm interop
RankLocator = 5 // opaque locators — never outrank a real digest RankMidhash = 10 // the record address — internally meaningful, externally invisible
RankUnknown = 0 // weak/unknown digests, unparseable input RankLocator = 5 // opaque locators — never outrank a real digest
RankUnknown = 0 // weak/unknown digests, unparseable input
) )
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content // Decode parses a multibase-base32 CIDv1 string ("b…") into its content
@@ -112,11 +133,16 @@ func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network. // - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS // - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
// block of the file IS its sha2-256, so they share this tier). // block of the file IS its sha2-256, so they share this tier).
// - nzb-posting is content-derived (a digest over a Usenet posting's article
// Message-ID set) so it outranks the opaque locators below, but the
// digest addresses no fetchable block, so it stays below a real content
// digest — once the media is materialised and re-seeded under a genuine
// sha2-256 cid, that cid must win the election, not this one.
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for // - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
// swarm interop, not strength. // swarm interop, not strength.
// - midhash256 is the floor among real *digests*: internally meaningful, // - midhash256 is the floor among real *digests*: internally meaningful,
// externally invisible. // externally invisible.
// - opaque locators (nzb-release, url) sit BELOW midhash. They are not // - opaque locators (nzb-release, url, card) sit BELOW midhash. They are not
// content-derived — two URLs to the same bytes yield two different locator // content-derived — two URLs to the same bytes yield two different locator
// CIDs — so a locator must never displace a real digest as the identity a // CIDs — so a locator must never displace a real digest as the identity a
// record is advertised under. Tier 5 means a locator wins the election only // record is advertised under. Tier 5 means a locator wins the election only
@@ -139,9 +165,15 @@ func Rank(cidStr string) int {
} }
// Opaque locators. Codec-only: their multihash is identity (0x00), so // Opaque locators. Codec-only: their multihash is identity (0x00), so
// there is nothing to match on the mh side. // there is nothing to match on the mh side.
if codec == CodeNzbRelease || codec == CodeURL { if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard {
return RankLocator return RankLocator
} }
// nzb-posting: codec-only, checked BEFORE the mh switch below — its
// multihash is a real sha2-256 and would otherwise match CodeSha256 there
// and rank 30, tying with genuine content digests.
if codec == CodeNzbPosting {
return RankNzbPosting
}
switch mh { switch mh {
case CodeSha256, CodeSha3_256, CodeSha3_384: case CodeSha256, CodeSha3_256, CodeSha3_384:
return RankDigest return RankDigest
+22 -3
View File
@@ -150,12 +150,31 @@ func TestRank_Tiers(t *testing.T) {
func TestRank_StrictOrdering(t *testing.T) { func TestRank_StrictOrdering(t *testing.T) {
ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32)) ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
sha := Rank(cidSha256) sha := Rank(cidSha256)
posting := Rank(buildCID(CodeNzbPosting, CodeSha256, 32))
btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32)) btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32)) mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
unknown := Rank(cidMd5) unknown := Rank(cidMd5)
if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) { if !(ipfs > sha && sha > posting && posting > btih && btih > mid && mid > unknown) {
t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d", t.Fatalf("rank ordering broken: ipfs=%d sha=%d posting=%d btih=%d mid=%d unknown=%d",
ipfs, sha, btih, mid, unknown) ipfs, sha, posting, btih, mid, unknown)
}
}
// TestRank_NzbPostingMatchesCodecNotMultihash pins the inverse trap:
// nzb-posting's multihash is a REAL sha2-256 (unlike nzb-release/url, whose mh
// is identity 0x00), so Rank must match on the codec BEFORE falling into the
// mh switch — otherwise this ties with a genuine content digest at 30 instead
// of sitting at 25.
func TestRank_NzbPostingMatchesCodecNotMultihash(t *testing.T) {
posting := buildCID(CodeNzbPosting, CodeSha256, 32)
if got := Rank(posting); got != RankNzbPosting {
t.Errorf("Rank(nzb-posting) = %d, want %d (RankNzbPosting) — codec check must precede the mh switch", got, RankNzbPosting)
}
// A real sha2-256 with the SAME digest bytes must still rank 30 — the
// codec, not the digest, is what distinguishes the two.
realDigest := buildCID(CodecRaw, CodeSha256, 32)
if got := Rank(realDigest); got != RankDigest {
t.Errorf("Rank(sha2-256) regressed to %d, want %d", got, RankDigest)
} }
} }