remove in metadata cid_* to put in the cids/ set
This commit is contained in:
Pierre
+108
-63
@@ -1,81 +1,126 @@
|
||||
// Package cid centralizes CID-format helpers: ranking by utility for
|
||||
// external presentation, normalizing prefixed CID tokens, and the rules for
|
||||
// picking which CID is the canonical externally-visible identifier of a
|
||||
// file.
|
||||
// Package cid centralizes CID-format helpers: decoding a multibase-base32
|
||||
// CIDv1 into its multicodec parts, ranking by utility for external
|
||||
// presentation, and picking which CID is the canonical externally-visible
|
||||
// identifier of a file.
|
||||
//
|
||||
// A "CID token" in this codebase is the prefixed form "<algorithm>:<value>",
|
||||
// e.g. "midhash256:bafkr4ih...", "sha256:bafkrei...", "ipfs:bafy...". The
|
||||
// prefix makes the type recoverable without parsing the multicodec, which is
|
||||
// what the reverse index (cid:<token> → uuid), the per-root cids set, and
|
||||
// the canonical_cid field all rely on.
|
||||
// Sibling CIDs are stored as a bare-CID key-set (`cids/<cid> = "true"`); a
|
||||
// CID is self-describing — its algorithm is recoverable from the CIDv1
|
||||
// codec / multihash code, so there is no `<algorithm>:` prefix and no
|
||||
// per-algorithm field name. Ranking and selection parse the multicodec
|
||||
// rather than a string prefix.
|
||||
package cid
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Rank returns a score for a CID token. Higher = more preferred as the
|
||||
// public-facing "canonical" identifier for a file. Unknown algorithms rank
|
||||
// below all known ones (0) so a plugin that invents a new digest type
|
||||
// can't accidentally outrank an established one until it's registered here.
|
||||
// Multicodec / multihash codes used for ranking. Source of truth for the
|
||||
// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
|
||||
// METADATA_KEYS.md §2.
|
||||
const (
|
||||
CodecRaw = 0x55 // raw — IPFS content-addressed block
|
||||
CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block
|
||||
CodeSha1 = 0x11 // multihash sha1
|
||||
CodeSha256 = 0x12 // multihash sha2-256
|
||||
CodeSha3_384 = 0x15 // multihash sha3-384
|
||||
CodeSha3_256 = 0x16 // multihash sha3-256
|
||||
CodeMd5 = 0xd5 // multihash md5
|
||||
CodeCrc32 = 0x0132 // multihash crc32
|
||||
CodeMidhash256 = 0x1000 // custom — SHA-256(size‖middle 1MB)
|
||||
CodeBtPiecesRoot = 0xb702 // bittorrent v2 pieces root
|
||||
CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52)
|
||||
CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID
|
||||
CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID
|
||||
)
|
||||
|
||||
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content
|
||||
// codec and multihash function code. It is the inverse of the encoder in
|
||||
// internal/watcher/midhash.go. Returns an error for anything that isn't a
|
||||
// CIDv1 in base32lower (e.g. a CIDv0 "Qm…" or a non-CID string).
|
||||
func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
|
||||
if len(cidStr) < 2 || cidStr[0] != 'b' {
|
||||
return 0, 0, fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
|
||||
}
|
||||
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
|
||||
}
|
||||
version, n := binary.Uvarint(raw)
|
||||
if n <= 0 {
|
||||
return 0, 0, fmt.Errorf("cid %q: bad version varint", cidStr)
|
||||
}
|
||||
raw = raw[n:]
|
||||
if version != 1 {
|
||||
return 0, 0, fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
|
||||
}
|
||||
codec, n = binary.Uvarint(raw)
|
||||
if n <= 0 {
|
||||
return 0, 0, fmt.Errorf("cid %q: bad codec varint", cidStr)
|
||||
}
|
||||
raw = raw[n:]
|
||||
mhCode, n = binary.Uvarint(raw)
|
||||
if n <= 0 {
|
||||
return 0, 0, fmt.Errorf("cid %q: bad multihash-code varint", cidStr)
|
||||
}
|
||||
return codec, mhCode, nil
|
||||
}
|
||||
|
||||
// Rank returns a score for a bare CID. Higher = more preferred as the
|
||||
// public-facing "canonical" identifier for a file. Unknown / undecodable
|
||||
// CIDs rank 0 so a new digest type can't accidentally outrank an
|
||||
// established one until it's classified here.
|
||||
//
|
||||
// We rank by the **multihash code** (the algorithm), with the custom
|
||||
// BitTorrent CID codecs checked alongside it. This is deliberate: MetaMesh's
|
||||
// two CID encoders disagree on the *content codec* — meta-hash/the watcher
|
||||
// set codec == algorithm code, while the fullhash plugin and gateway wrap
|
||||
// every digest with the raw codec (0x55) and put the algorithm in the
|
||||
// multihash code. The multihash code is the algorithm in all of them, so it
|
||||
// is the only reliable discriminator. (The BT info hash is given a distinct
|
||||
// multihash code — btih-v2 0x10B7 — so it can't be confused with a plain
|
||||
// sha2-256, see METADATA_KEYS.md §14.13.)
|
||||
//
|
||||
// Ordering rationale (utility-weighted, NOT cryptographic-strength):
|
||||
// - IPFS CIDs win because the network around them — gateways, swarm,
|
||||
// content-addressing tooling — has the broadest reach. A cryptographically
|
||||
// equivalent sha256 in a non-CID form has less downstream value.
|
||||
// - sha256 / sha3 follow: standard, widely recognized, no platform lock-in.
|
||||
// - BitTorrent infohashes (btih) outrank midhash256 not for strength
|
||||
// (v1 is sha1, weaker than midhash256's sha256 inner) but for swarm
|
||||
// interop — a btih makes a file discoverable on existing torrent
|
||||
// infrastructure that knows nothing about MetaMesh.
|
||||
// - midhash256 is the floor. Internally meaningful, externally invisible.
|
||||
func Rank(token string) int {
|
||||
switch AlgorithmOf(token) {
|
||||
case "ipfs":
|
||||
// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
|
||||
// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
|
||||
// block of the file IS its sha2-256, so they share this tier).
|
||||
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
|
||||
// swarm interop, not strength.
|
||||
// - midhash256 is the floor among real options: internally meaningful,
|
||||
// externally invisible.
|
||||
// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
|
||||
func Rank(cidStr string) int {
|
||||
codec, mh, err := Decode(cidStr)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
if codec == CodecDagPB {
|
||||
return 40
|
||||
case "sha256", "sha3", "sha3-256", "sha3-384", "sha3-512":
|
||||
return 30
|
||||
case "btih", "btih-v2":
|
||||
}
|
||||
// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
|
||||
// multihash code, in either encoder convention.
|
||||
if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
|
||||
mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
|
||||
return 20
|
||||
case "midhash256":
|
||||
}
|
||||
switch mh {
|
||||
case CodeSha256, CodeSha3_256, CodeSha3_384:
|
||||
return 30
|
||||
case CodeMidhash256:
|
||||
return 10
|
||||
default:
|
||||
// md5, crc32, plain sha1, pieces-root, unknown.
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// AlgorithmOf returns the algorithm prefix of a CID token, or "" if the
|
||||
// token is malformed (no colon, or empty prefix).
|
||||
func AlgorithmOf(token string) string {
|
||||
colon := strings.IndexByte(token, ':')
|
||||
if colon <= 0 {
|
||||
return ""
|
||||
}
|
||||
return token[:colon]
|
||||
}
|
||||
|
||||
// Token assembles a CID token from its algorithm and raw value. Mirror of
|
||||
// AlgorithmOf — callers writing to the reverse index, cids set, or
|
||||
// canonical_cid field route through this so the format stays consistent.
|
||||
func Token(algorithm, raw string) string {
|
||||
if algorithm == "" || raw == "" {
|
||||
return ""
|
||||
}
|
||||
return algorithm + ":" + raw
|
||||
}
|
||||
|
||||
// ValueOf returns the raw value portion of a CID token (everything after
|
||||
// the first colon). Returns "" for malformed tokens.
|
||||
func ValueOf(token string) string {
|
||||
colon := strings.IndexByte(token, ':')
|
||||
if colon <= 0 || colon == len(token)-1 {
|
||||
return ""
|
||||
}
|
||||
return token[colon+1:]
|
||||
}
|
||||
|
||||
// Better picks the higher-ranked of two tokens. On ties, returns the
|
||||
// Better picks the higher-ranked of two CIDs. On ties, returns the
|
||||
// lexicographically smaller one — gives the reconciler a deterministic
|
||||
// answer so an unchanged canonical_cid doesn't churn when peers add the
|
||||
// same digest under a different ordering.
|
||||
// answer so the chosen canonical doesn't churn when peers add the same
|
||||
// digest under a different ordering.
|
||||
func Better(a, b string) string {
|
||||
ra, rb := Rank(a), Rank(b)
|
||||
if ra > rb {
|
||||
|
||||
+107
-62
@@ -1,85 +1,130 @@
|
||||
package cid
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRank_Ordering(t *testing.T) {
|
||||
// buildCID assembles a multibase-base32 CIDv1 from a content codec, a
|
||||
// multihash code, and a digest length (digest bytes are zero — only the
|
||||
// header is under test). Mirrors the encoder in watcher/midhash.go and the
|
||||
// TS CID.createV1(code, create(code, digest)) path.
|
||||
func buildCID(codec, mhCode uint64, digestLen int) string {
|
||||
var b []byte
|
||||
b = binary.AppendUvarint(b, 1) // CIDv1
|
||||
b = binary.AppendUvarint(b, codec)
|
||||
b = binary.AppendUvarint(b, mhCode)
|
||||
b = binary.AppendUvarint(b, uint64(digestLen))
|
||||
b = append(b, make([]byte, digestLen)...)
|
||||
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
|
||||
return "b" + strings.ToLower(enc)
|
||||
}
|
||||
|
||||
// Real CIDs produced by @metazla/meta-hash (from its main.test.ts). For
|
||||
// digest CIDs meta-hash sets codec == multihash code == algorithm code.
|
||||
const (
|
||||
cidCrc32 = "bagzafmqcathsbnrk"
|
||||
cidMd5 = "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa"
|
||||
cidSha1 = "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra"
|
||||
cidSha256 = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
|
||||
cidSha3256 = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
|
||||
cidSha3384 = "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga"
|
||||
)
|
||||
|
||||
func TestDecode_Fixtures(t *testing.T) {
|
||||
cases := []struct {
|
||||
token string
|
||||
want int
|
||||
name string
|
||||
cid string
|
||||
wantMh uint64
|
||||
}{
|
||||
{"ipfs:bafy", 40},
|
||||
{"sha256:bafk", 30},
|
||||
{"sha3-256:abc", 30},
|
||||
{"btih:def", 20},
|
||||
{"btih-v2:def", 20},
|
||||
{"midhash256:bafk", 10},
|
||||
{"unknown:xyz", 0},
|
||||
{"", 0},
|
||||
{"bare-no-colon", 0},
|
||||
{"crc32", cidCrc32, CodeCrc32},
|
||||
{"md5", cidMd5, CodeMd5},
|
||||
{"sha1", cidSha1, CodeSha1},
|
||||
{"sha2-256", cidSha256, CodeSha256},
|
||||
{"sha3-256", cidSha3256, CodeSha3_256},
|
||||
{"sha3-384", cidSha3384, CodeSha3_384},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := Rank(tc.token); got != tc.want {
|
||||
t.Errorf("Rank(%q) = %d, want %d", tc.token, got, tc.want)
|
||||
codec, mh, err := Decode(tc.cid)
|
||||
if err != nil {
|
||||
t.Errorf("Decode(%s %q) error: %v", tc.name, tc.cid, err)
|
||||
continue
|
||||
}
|
||||
// meta-hash digest CIDs use codec == multihash code.
|
||||
if codec != tc.wantMh || mh != tc.wantMh {
|
||||
t.Errorf("Decode(%s) = codec 0x%x mh 0x%x, want both 0x%x", tc.name, codec, mh, tc.wantMh)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecode_Rejects(t *testing.T) {
|
||||
for _, in := range []string{"", "Qmabc", "x", "bzzzz!!"} {
|
||||
if _, _, err := Decode(in); err == nil {
|
||||
t.Errorf("Decode(%q) expected error, got nil", in)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRank_Tiers(t *testing.T) {
|
||||
ipfsDagPB := buildCID(CodecDagPB, CodeSha256, 32) // chunked IPFS file
|
||||
rawSha256 := buildCID(CodecRaw, CodeSha256, 32) // fullhash-style sha2-256
|
||||
btihV2meta := buildCID(CodeBtihV2, CodeBtihV2, 32) // meta-hash btih-v2 (codec==mh)
|
||||
btihV2raw := buildCID(CodecRaw, CodeBtihV2, 32) // fullhash btih-v2 (raw codec)
|
||||
btV1File := buildCID(CodeBtihV1File, CodeBtihV1File, 20) // gateway per-file btih v1
|
||||
midhashRaw := buildCID(CodecRaw, CodeMidhash256, 32) // fullhash midhash (raw codec)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
cid string
|
||||
want int
|
||||
}{
|
||||
{"ipfs-dagpb", ipfsDagPB, 40},
|
||||
{"sha2-256(fixture, codec==mh)", cidSha256, 30},
|
||||
{"sha2-256(raw codec)", rawSha256, 30},
|
||||
{"sha3-256", cidSha3256, 30},
|
||||
{"sha3-384", cidSha3384, 30},
|
||||
{"btih-v2(codec==mh)", btihV2meta, 20},
|
||||
{"btih-v2(raw codec)", btihV2raw, 20},
|
||||
{"btih-v1-file", btV1File, 20},
|
||||
{"midhash256(raw codec)", midhashRaw, 10},
|
||||
{"sha1", cidSha1, 0},
|
||||
{"md5", cidMd5, 0},
|
||||
{"crc32", cidCrc32, 0},
|
||||
{"garbage", "not-a-cid", 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := Rank(tc.cid); got != tc.want {
|
||||
t.Errorf("Rank(%s) = %d, want %d", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRank_StrictOrdering(t *testing.T) {
|
||||
// The exact numbers don't matter, but the inequality chain does — this
|
||||
// is what the canonical_cid reconciler relies on.
|
||||
ipfs := Rank("ipfs:x")
|
||||
sha := Rank("sha256:x")
|
||||
btih := Rank("btih:x")
|
||||
mid := Rank("midhash256:x")
|
||||
unknown := Rank("unknown:x")
|
||||
|
||||
ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
|
||||
sha := Rank(cidSha256)
|
||||
btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
|
||||
mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
|
||||
unknown := Rank(cidMd5)
|
||||
if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
|
||||
t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
|
||||
ipfs, sha, btih, mid, unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlgorithmOf(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"ipfs:bafy": "ipfs",
|
||||
"sha256:bafk": "sha256",
|
||||
"midhash256:abc": "midhash256",
|
||||
"": "",
|
||||
"no-colon": "",
|
||||
":leading-colon": "", // empty algo not allowed
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := AlgorithmOf(in); got != want {
|
||||
t.Errorf("AlgorithmOf(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestToken(t *testing.T) {
|
||||
if got := Token("sha256", "abc"); got != "sha256:abc" {
|
||||
t.Errorf("Token(sha256, abc) = %q", got)
|
||||
}
|
||||
if got := Token("", "abc"); got != "" {
|
||||
t.Errorf("Token with empty algo should return empty, got %q", got)
|
||||
}
|
||||
if got := Token("sha256", ""); got != "" {
|
||||
t.Errorf("Token with empty value should return empty, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBetter(t *testing.T) {
|
||||
// IPFS beats sha256
|
||||
if got := Better("ipfs:a", "sha256:b"); got != "ipfs:a" {
|
||||
t.Errorf("Better(ipfs, sha256) = %q, want ipfs:a", got)
|
||||
dagpb := buildCID(CodecDagPB, CodeSha256, 32)
|
||||
// dag-pb (IPFS) beats a sha2-256 digest, regardless of arg order.
|
||||
if got := Better(dagpb, cidSha256); got != dagpb {
|
||||
t.Errorf("Better(dagpb, sha256) = %q, want dagpb", got)
|
||||
}
|
||||
if got := Better("sha256:b", "ipfs:a"); got != "ipfs:a" {
|
||||
t.Errorf("Better(sha256, ipfs) = %q, want ipfs:a (order shouldn't matter)", got)
|
||||
if got := Better(cidSha256, dagpb); got != dagpb {
|
||||
t.Errorf("Better(sha256, dagpb) = %q, want dagpb (order shouldn't matter)", got)
|
||||
}
|
||||
// Tie → lexicographically smaller wins, deterministic
|
||||
if got := Better("ipfs:b", "ipfs:a"); got != "ipfs:a" {
|
||||
t.Errorf("Better(ipfs:b, ipfs:a) = %q, want ipfs:a", got)
|
||||
}
|
||||
if got := Better("ipfs:a", "ipfs:b"); got != "ipfs:a" {
|
||||
t.Errorf("Better(ipfs:a, ipfs:b) = %q, want ipfs:a", got)
|
||||
// Tie → lexicographically smaller wins, deterministic.
|
||||
a, b := "bafyaaa", "bafyaab"
|
||||
if Better(a, b) != a || Better(b, a) != a {
|
||||
t.Errorf("tie-break not deterministic for %q vs %q", a, b)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user