This commit is contained in:
Worph
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"$schema": "cid-rank-vectors/v1",
|
||||
"$comment": [
|
||||
"CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
|
||||
"The rank decides which of a record's several CIDs is elected canonical - the cid a",
|
||||
"record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
|
||||
"they cannot share code (independent submodules, no cargo path-dep). These vectors are",
|
||||
"the only thing holding them in sync: every implementation's test suite loads this file.",
|
||||
"",
|
||||
"This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
|
||||
"own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
|
||||
"",
|
||||
"Ranking is by UTILITY for external retrieval, not cryptographic strength.",
|
||||
"Ties break on the lexicographically smaller cid, so the election is deterministic",
|
||||
"across peers and does not churn.",
|
||||
"",
|
||||
"See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
|
||||
],
|
||||
"ladder": {
|
||||
"40": "dag-pb (IPFS block)",
|
||||
"30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
|
||||
"20": "BitTorrent infohashes - swarm interop",
|
||||
"10": "midhash256 - the record address",
|
||||
"5": "opaque locators (nzb-release, url) - never outrank a real digest",
|
||||
"0": "weak/unknown digests, unparseable input"
|
||||
},
|
||||
"tieBreak": "lexicographically smaller cid wins",
|
||||
"vectors": [
|
||||
{
|
||||
"name": "ipfs-dagpb",
|
||||
"cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 40,
|
||||
"note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
|
||||
},
|
||||
{
|
||||
"name": "sha2-256-raw-codec",
|
||||
"cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 30,
|
||||
"note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
|
||||
},
|
||||
{
|
||||
"name": "sha2-256-codec-eq-mh",
|
||||
"cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture; meta-hash sets codec == mh"
|
||||
},
|
||||
{
|
||||
"name": "sha3-256",
|
||||
"cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture"
|
||||
},
|
||||
{
|
||||
"name": "sha3-384",
|
||||
"cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
|
||||
"rank": 30,
|
||||
"note": "REAL meta-hash fixture"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-codec-eq-mh",
|
||||
"cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "meta-hash encoder convention"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-raw-codec",
|
||||
"cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "fullhash encoder convention - SAME tier, both conventions must match"
|
||||
},
|
||||
{
|
||||
"name": "btih-v1-file",
|
||||
"cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "gateway per-file BitTorrent v1"
|
||||
},
|
||||
{
|
||||
"name": "btih-v2-file",
|
||||
"cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 20,
|
||||
"note": "gateway per-file BitTorrent v2"
|
||||
},
|
||||
{
|
||||
"name": "midhash256-codec-eq-mh",
|
||||
"cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 10,
|
||||
"note": "the record address; internally meaningful, externally invisible"
|
||||
},
|
||||
{
|
||||
"name": "midhash256-raw-codec",
|
||||
"cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 10
|
||||
},
|
||||
{
|
||||
"name": "nzb-release-locator",
|
||||
"cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
|
||||
"rank": 5,
|
||||
"note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
|
||||
},
|
||||
{
|
||||
"name": "url-locator",
|
||||
"cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
|
||||
"rank": 5,
|
||||
"note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
|
||||
},
|
||||
{
|
||||
"name": "sha1",
|
||||
"cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "md5",
|
||||
"cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "crc32",
|
||||
"cid": "bagzafmqcathsbnrk",
|
||||
"rank": 0,
|
||||
"note": "REAL meta-hash fixture; weak digest"
|
||||
},
|
||||
{
|
||||
"name": "bt-pieces-root",
|
||||
"cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 0,
|
||||
"note": "torrent-internal merkle root, NOT a file-identity cid"
|
||||
},
|
||||
{
|
||||
"name": "cidv0-rejected",
|
||||
"cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
|
||||
"rank": 0,
|
||||
"note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
|
||||
},
|
||||
{
|
||||
"name": "token-form-rejected",
|
||||
"cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
"rank": 0,
|
||||
"note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
|
||||
},
|
||||
{
|
||||
"name": "garbage",
|
||||
"cid": "not-a-cid",
|
||||
"rank": 0
|
||||
},
|
||||
{
|
||||
"name": "empty",
|
||||
"cid": "",
|
||||
"rank": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
+43
-7
@@ -20,6 +20,14 @@ import (
|
||||
// Multicodec / multihash codes used for ranking. Source of truth for the
|
||||
// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
|
||||
// METADATA_KEYS.md §2.
|
||||
//
|
||||
// The tier every code maps to is pinned by the golden fixture
|
||||
// `cid-rank-vectors.json` (vendored next to this package). This is one of
|
||||
// seven implementations of the same ladder across Go, Rust and TypeScript;
|
||||
// the fixture is the only thing keeping them in agreement. If you add a code
|
||||
// here, add a vector — an unranked code silently falls to tier 0, which is
|
||||
// exactly how the nzb-release/url locators came to be ranked 20 in meta-share
|
||||
// and 0 here and in meta-search.
|
||||
const (
|
||||
CodecRaw = 0x55 // raw — IPFS content-addressed block
|
||||
CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block
|
||||
@@ -34,6 +42,23 @@ const (
|
||||
CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52)
|
||||
CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID
|
||||
CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID
|
||||
|
||||
// Opaque locators. These are NOT digests: the payload rides in an identity
|
||||
// multihash (0x00), so the code identifying them sits in the *content
|
||||
// codec* slot, never the multihash slot. Rank must match on the codec for
|
||||
// these — matching on mh would see 0x00 and rank them 0.
|
||||
CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
|
||||
CodeURL = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
|
||||
)
|
||||
|
||||
// Rank tiers, named so the ladder reads as a ladder.
|
||||
const (
|
||||
RankIPFS = 40 // dag-pb — broadest network reach
|
||||
RankDigest = 30 // sha2-256 / sha3-* — real content digests
|
||||
RankBtih = 20 // BitTorrent infohashes — swarm interop
|
||||
RankMidhash = 10 // the record address — internally meaningful, externally invisible
|
||||
RankLocator = 5 // opaque locators — never outrank a real digest
|
||||
RankUnknown = 0 // weak/unknown digests, unparseable input
|
||||
)
|
||||
|
||||
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content
|
||||
@@ -89,31 +114,42 @@ func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
|
||||
// block of the file IS its sha2-256, so they share this tier).
|
||||
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
|
||||
// swarm interop, not strength.
|
||||
// - midhash256 is the floor among real options: internally meaningful,
|
||||
// - midhash256 is the floor among real *digests*: internally meaningful,
|
||||
// externally invisible.
|
||||
// - opaque locators (nzb-release, url) sit BELOW midhash. They are not
|
||||
// content-derived — two URLs to the same bytes yield two different locator
|
||||
// CIDs — so a locator must never displace a real digest as the identity a
|
||||
// record is advertised under. Tier 5 means a locator wins the election only
|
||||
// when it is the record's sole CID, which is exactly the guarantee
|
||||
// METADATA_KEYS.md §2 states.
|
||||
// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
|
||||
func Rank(cidStr string) int {
|
||||
codec, mh, err := Decode(cidStr)
|
||||
if err != nil {
|
||||
return 0
|
||||
return RankUnknown
|
||||
}
|
||||
if codec == CodecDagPB {
|
||||
return 40
|
||||
return RankIPFS
|
||||
}
|
||||
// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
|
||||
// multihash code, in either encoder convention.
|
||||
if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
|
||||
mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
|
||||
return 20
|
||||
return RankBtih
|
||||
}
|
||||
// Opaque locators. Codec-only: their multihash is identity (0x00), so
|
||||
// there is nothing to match on the mh side.
|
||||
if codec == CodeNzbRelease || codec == CodeURL {
|
||||
return RankLocator
|
||||
}
|
||||
switch mh {
|
||||
case CodeSha256, CodeSha3_256, CodeSha3_384:
|
||||
return 30
|
||||
return RankDigest
|
||||
case CodeMidhash256:
|
||||
return 10
|
||||
return RankMidhash
|
||||
default:
|
||||
// md5, crc32, plain sha1, pieces-root, unknown.
|
||||
return 0
|
||||
return RankUnknown
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,10 +3,56 @@ package cid
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// goldenVectors is the CANONICAL cross-language fixture: the same file is
|
||||
// vendored into meta-search, meta-share and meta-hash, and each of their test
|
||||
// suites asserts against it. It is the only mechanism keeping the seven
|
||||
// implementations of this ladder in agreement — they live in independent
|
||||
// submodules and cannot share code. `scripts/check-cid-vectors.sh` at the
|
||||
// meta-root fails if any vendored copy drifts from the canonical one.
|
||||
const goldenVectors = "cid-rank-vectors.json"
|
||||
|
||||
type vector struct {
|
||||
Name string `json:"name"`
|
||||
CID string `json:"cid"`
|
||||
Rank int `json:"rank"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
func loadVectors(t *testing.T) []vector {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(goldenVectors)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v (run ./scripts/check-cid-vectors.sh --fix from the meta-root)", goldenVectors, err)
|
||||
}
|
||||
var doc struct {
|
||||
Vectors []vector `json:"vectors"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
t.Fatalf("parse %s: %v", goldenVectors, err)
|
||||
}
|
||||
if len(doc.Vectors) == 0 {
|
||||
t.Fatalf("%s has no vectors", goldenVectors)
|
||||
}
|
||||
return doc.Vectors
|
||||
}
|
||||
|
||||
// TestRank_GoldenVectors is the load-bearing test. Every other test in this
|
||||
// file is a convenience; this one is the contract with Rust and TypeScript.
|
||||
func TestRank_GoldenVectors(t *testing.T) {
|
||||
for _, v := range loadVectors(t) {
|
||||
if got := Rank(v.CID); got != v.Rank {
|
||||
t.Errorf("Rank(%s) = %d, want %d\n cid: %s\n note: %s",
|
||||
v.Name, got, v.Rank, v.CID, v.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// buildCID assembles a multibase-base32 CIDv1 from a content codec, a
|
||||
// multihash code, and a digest length (digest bytes are zero — only the
|
||||
// header is under test). Mirrors the encoder in watcher/midhash.go and the
|
||||
|
||||
Reference in New Issue
Block a user