package main import ( "bytes" "database/sql" "encoding/base64" "encoding/binary" "encoding/json" "fmt" "math" "os" "path/filepath" "slices" "testing" "time" "github.com/metazla/meta-core/internal/claim" "github.com/metazla/meta-core/internal/identity" "github.com/metazla/meta-core/internal/storage" ) const ( cidIPFS = "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 40 cidSHA = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 30 cidB = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua" cidOrph = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq" ridA = "01JKR8XW5T4QMV7AHNJ2DEFGHK" ridB = "01JKR8XW5T4QMV7AHNJ2DEFGHM" ridC = "01JKR8XW5T4QMV7AHNJ2DEFGHN" ridD = "01JKR8XW5T4QMV7AHNJ2DEFGHP" oldTSsec = 1_700_000_000 ) // --- minimal canonical CBOR encoder for building MMUR1 fixtures --- func head(major byte, n uint64) []byte { m := major << 5 switch { case n < 24: return []byte{m | byte(n)} case n <= 0xff: return []byte{m | 24, byte(n)} case n <= 0xffff: return []byte{m | 25, byte(n >> 8), byte(n)} case n <= 0xffffffff: return binary.BigEndian.AppendUint32([]byte{m | 26}, uint32(n)) } return binary.BigEndian.AppendUint64([]byte{m | 27}, n) } func encText(s string) []byte { return append(head(3, uint64(len(s))), s...) } func encBytes(b []byte) []byte { return append(head(2, uint64(len(b))), b...) } func encUint(n uint64) []byte { return head(0, n) } func encBool(b bool) []byte { if b { return []byte{0xf5} } return []byte{0xf4} } func encF64(f float64) []byte { return binary.BigEndian.AppendUint64([]byte{0xfb}, math.Float64bits(f)) } var encNull = []byte{0xf6} // mmur1 builds a signed MMUR1 record; badSig signs other bytes. func mmur1(t *testing.T, id *identity.Identity, cid, key string, value []byte, version uint64, badSig bool) []byte { t.Helper() var elems []byte elems = append(elems, encText(cid)...) elems = append(elems, encText(id.UID)...) elems = append(elems, encText(key)...) elems = append(elems, value...) elems = append(elems, encUint(version)...) elems = append(elems, encUint(oldTSsec)...) signed := append(append([]byte(mmur1Domain), 0x86), elems...) if badSig { signed = append(signed, 'x') } sig, err := identity.Sign(id, signed) if err != nil { t.Fatal(err) } return append(append([]byte{0x87}, elems...), encBytes(sig)...) } func cell(t *testing.T, rec []byte, version uint64) string { b, err := json.Marshal(map[string]any{ "record": base64.StdEncoding.EncodeToString(rec), "version": version, "ts": oldTSsec, }) if err != nil { t.Fatal(err) } return string(b) } func importID(t *testing.T, n byte) *identity.Identity { t.Helper() id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 7)) if err != nil { t.Fatal(err) } return id } type fixture struct { dir, oldPath, accounts string user, stranger *identity.Identity seekPlain, durPlain []byte } func buildV1(t *testing.T) fixture { t.Helper() dir := t.TempDir() f := fixture{dir: dir, oldPath: filepath.Join(dir, "v1.db"), accounts: filepath.Join(dir, "accounts")} f.user, f.stranger = importID(t, 0x11), importID(t, 0x22) if err := os.MkdirAll(f.accounts, 0o700); err != nil { t.Fatal(err) } if err := identity.SaveAccount(f.accounts, f.user); err != nil { t.Fatal(err) } aead, err := identity.DeriveAEADKey(f.user) if err != nil { t.Fatal(err) } f.seekPlain = encF64(123.5) f.durPlain = encF64(5400) env, err := sealEnvelope(aead, f.seekPlain, envelopeAAD(cidIPFS, f.user.UID, "seek", 3), bytes.Repeat([]byte{9}, 24)) if err != nil { t.Fatal(err) } db, err := sql.Open("sqlite", "file:"+f.oldPath) if err != nil { t.Fatal(err) } defer db.Close() if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { t.Fatal(err) } u := f.user.UID rows := map[string]string{ "meta-core:schema-version": "2", // record A: two cids, canonical = the dag-pb one "file:" + ridA + "/cids/" + cidIPFS: "true", "file:" + ridA + "/cids/" + cidSHA: "true", "file:" + ridA + "/title": "Inception", "file:" + ridA + "/year": "007", "file:" + ridA + "/sizeByte": "15032385536", "file:" + ridA + "/genres/Drama": "true", "file:" + ridA + "/filePath": "/files/watch/Inception.mkv", "file:" + ridA + "/mtimeNano": "1729872000000000000", "file:" + ridA + "/cid_sha256": cidSHA, "file:" + ridA + "/fileinfo/duration": "8878.5", "cid:" + cidIPFS: ridA, "cid:" + cidSHA: ridA, // record B: its cid only in the index; claims cidSHA too (conflict) "file:" + ridB + "/title": "Other", "file:" + ridB + "/cids/" + cidSHA: "true", "cid:" + cidB: ridB, // record C: no cid at all → dropped "file:" + ridC + "/title": "Nobody", // orphan index row "cid:" + cidOrph: ridD, // scratch + derived "btmagnet:abc": "magnet:?xt=urn:btih:abc", "searchcov:q": "1", "nzb:x": "y", "idx:roots/" + ridA: "", "dup:" + ridA + "/p": "", "udl:idx:user:" + u + "\x1f": "", // UDL "udl:rec:" + u + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.user, cidIPFS, "like", encBool(true), 1, false), 1), "udl:rec:" + u + "/" + cidIPFS + "/rating": cell(t, mmur1(t, f.user, cidIPFS, "rating", encUint(4), 1, false), 1), "udl:rec:" + u + "/" + cidIPFS + "/seek": cell(t, mmur1(t, f.user, cidIPFS, "seek", encBytes(env), 3, false), 3), "udl:rec:" + u + "/" + cidIPFS + "/watched": cell(t, mmur1(t, f.user, cidIPFS, "watched", encNull, 2, false), 2), "udl:rec:" + u + "/" + cidIPFS + "/duration": cell(t, mmur1(t, f.user, cidIPFS, "duration", f.durPlain, 1, false), 1), "udl:rec:" + u + "/self/profile:name": cell(t, mmur1(t, f.user, "self", "profile:name", encText("Alice"), 1, false), 1), "udl:rec:" + u + "/" + cidSHA + "/like": cell(t, mmur1(t, f.user, cidSHA, "like", encBool(true), 1, true), 1), "udl:rec:" + f.stranger.UID + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.stranger, cidIPFS, "like", encBool(false), 1, false), 1), } for k, v := range rows { if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil { t.Fatal(err) } } return f } func TestMigrate(t *testing.T) { f := buildV1(t) truthPath := filepath.Join(f.dir, "truth.db") now := time.UnixMilli(1_790_000_000_000) st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, now: now, workers: 3}) if err != nil { t.Fatalf("migrate: %v", err) } var sum bytes.Buffer printSummary(&sum, st, false) t.Log(sum.String()) check := func(name string, got, want int) { t.Helper() if got != want { t.Errorf("%s = %d, want %d", name, got, want) } } check("rids", st.Rids, 2) check("records without cid", st.RecordsNoCID, 1) check("orphan cid rows", st.OrphanCIDRows, 1) check("cid conflicts", st.CIDConflicts, 1) check("alias claims", st.AliasClaims, 1) check("field claims", st.FieldClaims, 6) // A: title year sizeByte genres/Drama fileinfo/duration; B: title check("refused cid_*", st.RefusedFields["cid_*"], 1) check("watcher facts", st.WatcherFacts.total(), 2) check("udl migrated", st.UDLMigrated.total(), 6) check("udl no account", st.UDLDropped["no account on this box"], 1) check("udl bad sig", st.UDLDropped["bad MMUR1 signature"], 1) check("groups minted", st.GroupsMinted, 1) // "self" check("cid_group rows", st.CIDGroupRows, 4) check("scratch", st.Scratch.total(), 3) check("principals", st.Principals, 2) check("claims", st.ClaimsWritten, st.FieldClaims+st.AliasClaims+st.UDLMigrated.total()) // Reopen with meta-core's storage package: boot catch-up rebuilds the cache. c := storage.NewClient() if err := c.Connect(truthPath, filepath.Join(f.dir, "cache.db")); err != nil { t.Fatal(err) } defer c.Close() if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil { t.Fatal(err) } if err := c.Start(); err != nil { t.Fatal(err) } for cid, want := range map[string]string{cidIPFS: ridA, cidSHA: ridA, cidB: ridB, cidOrph: ""} { got, err := c.GetByCID(cid) if err != nil || got != want { t.Errorf("GetByCID(%s) = %q, %v; want %q", cid, got, err, want) } } a, err := c.GetMetadataFlat(ridA) if err != nil { t.Fatal(err) } for k, want := range map[string]string{ "title": "Inception", "year": "007", "sizeByte": "15032385536", "genres/Drama": "true", "fileinfo/duration": "8878.5", "cids/" + cidIPFS: "true", "cids/" + cidSHA: "true", } { if a[k] != want { t.Errorf("record A %s = %q, want %q", k, a[k], want) } } for _, k := range []string{"filePath", "mtimeNano", "cid_sha256"} { if _, ok := a[k]; ok { t.Errorf("record A carries %s", k) } } b, err := c.GetMetadataFlat(ridB) if err != nil || b["title"] != "Other" { t.Errorf("record B = %v, %v", b, err) } ids, err := c.GetAllHashIDs() if err != nil { t.Fatal(err) } if !slices.Contains(ids, ridA) || !slices.Contains(ids, ridB) || slices.Contains(ids, ridC) || slices.Contains(ids, ridD) { t.Errorf("roots = %v", ids) } if len(ids) != 3 { t.Errorf("want 3 roots (A, B, self), got %v", ids) } // Legacy claims sit on the canonical cid, signed by the printed uid. cls, err := c.ClaimsForCIDs([]string{cidIPFS}, "title") if err != nil || len(cls) != 1 || cls[0].UID != st.LegacyUID || cls[0].TS != uint64(now.UnixMilli()) { t.Fatalf("title claims = %+v, %v", cls, err) } if err := cls[0].Verify(); err != nil { t.Error(err) } alias, err := c.ClaimsForCIDs([]string{cidIPFS}, "cids/"+cidSHA) if err != nil || len(alias) != 1 || alias[0].Value.Text != "true" { t.Errorf("alias claims = %+v, %v", alias, err) } // UDL claims. u := f.user.UID wantTS := uint64(oldTSsec) * 1000 get := func(cid, key string) claim.Claim { t.Helper() cl, ok, err := c.UDLGetClaim(u, cid, key) if err != nil || !ok { t.Fatalf("UDL %s/%s: ok=%v err=%v", cid, key, ok, err) } if cl.TS != wantTS { t.Errorf("UDL %s ts = %d, want %d", key, cl.TS, wantTS) } if err := cl.Verify(); err != nil { t.Errorf("UDL %s: %v", key, err) } return cl } if v := get(cidIPFS, "like").Value; v.Kind != claim.KindText || v.Text != "true" { t.Errorf("like = %+v", v) } if v := get(cidIPFS, "userRating").Value; v.Kind != claim.KindText || v.Text != "4" { t.Errorf("userRating = %+v", v) } if _, ok, _ := c.UDLGetClaim(u, cidIPFS, "rating"); ok { t.Error("rating was not renamed") } if v := get(cidIPFS, "watched").Value; !v.IsNull() { t.Errorf("watched tombstone = %+v", v) } if v := get("self", "profile:name").Value; v.Text != "Alice" { t.Errorf("profile:name = %+v", v) } aead, _ := identity.DeriveAEADKey(f.user) for key, plain := range map[string][]byte{"seek": f.seekPlain, "duration": f.durPlain} { cl := get(cidIPFS, key) if cl.Value.Kind != claim.KindBytes { t.Fatalf("%s is not an envelope: %+v", key, cl.Value) } pt, err := openEnvelope(aead, cl.Value.Bytes, envelopeAAD(cidIPFS, u, key, cl.TS)) if err != nil || !bytes.Equal(pt, plain) { t.Errorf("%s decrypts to %x, %v; want %x", key, pt, err, plain) } } if _, ok, _ := c.UDLGetClaim(f.stranger.UID, cidIPFS, "like"); ok { t.Error("stranger's cell migrated without an account") } // A second run refuses to overwrite. if _, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts}); err == nil { t.Error("second run overwrote the truth file") } } func TestDryRunWritesNothing(t *testing.T) { f := buildV1(t) truthPath := filepath.Join(f.dir, "truth.db") st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, dryRun: true}) if err != nil { t.Fatal(err) } if _, err := os.Stat(truthPath); !os.IsNotExist(err) { t.Errorf("dry run created %s", truthPath) } if st.ClaimsWritten != st.FieldClaims+st.AliasClaims+st.UDLMigrated.total() || st.ClaimsWritten == 0 { t.Errorf("dry-run counts: %+v", st) } } func TestNoAccountsDropsUDL(t *testing.T) { f := buildV1(t) st, err := migrate(options{oldPath: f.oldPath, dryRun: true}) if err != nil { t.Fatal(err) } if st.UDLMigrated.total() != 0 || st.UDLDropped["no account on this box"] != 7 { t.Errorf("udl: migrated %v dropped %v", st.UDLMigrated, st.UDLDropped) } } func TestHalfFloat(t *testing.T) { for h, want := range map[uint16]float64{0x3c00: 1, 0xc000: -2, 0x3e00: 1.5, 0x0000: 0, 0x7bff: 65504} { if got := halfToFloat(h); got != want { t.Errorf("half %04x = %v, want %v", h, got, want) } } }