444 lines
15 KiB
Go
444 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"database/sql"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/metazla/meta-core/internal/claim"
|
|
"github.com/metazla/meta-core/internal/identity"
|
|
"github.com/metazla/meta-core/internal/storage"
|
|
)
|
|
|
|
const (
|
|
cidIPFS = "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 40
|
|
cidSHA = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 30
|
|
cidB = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
|
|
cidOrph = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
|
|
ridA = "01JKR8XW5T4QMV7AHNJ2DEFGHK"
|
|
ridB = "01JKR8XW5T4QMV7AHNJ2DEFGHM"
|
|
ridC = "01JKR8XW5T4QMV7AHNJ2DEFGHN"
|
|
ridD = "01JKR8XW5T4QMV7AHNJ2DEFGHP"
|
|
oldTSsec = 1_700_000_000
|
|
)
|
|
|
|
// --- minimal canonical CBOR encoder for building MMUR1 fixtures ---
|
|
|
|
func head(major byte, n uint64) []byte {
|
|
m := major << 5
|
|
switch {
|
|
case n < 24:
|
|
return []byte{m | byte(n)}
|
|
case n <= 0xff:
|
|
return []byte{m | 24, byte(n)}
|
|
case n <= 0xffff:
|
|
return []byte{m | 25, byte(n >> 8), byte(n)}
|
|
case n <= 0xffffffff:
|
|
return binary.BigEndian.AppendUint32([]byte{m | 26}, uint32(n))
|
|
}
|
|
return binary.BigEndian.AppendUint64([]byte{m | 27}, n)
|
|
}
|
|
|
|
func encText(s string) []byte { return append(head(3, uint64(len(s))), s...) }
|
|
func encBytes(b []byte) []byte { return append(head(2, uint64(len(b))), b...) }
|
|
func encUint(n uint64) []byte { return head(0, n) }
|
|
func encBool(b bool) []byte {
|
|
if b {
|
|
return []byte{0xf5}
|
|
}
|
|
return []byte{0xf4}
|
|
}
|
|
func encF64(f float64) []byte {
|
|
return binary.BigEndian.AppendUint64([]byte{0xfb}, math.Float64bits(f))
|
|
}
|
|
|
|
var encNull = []byte{0xf6}
|
|
|
|
// mmur1 builds a signed MMUR1 record; badSig signs other bytes.
|
|
func mmur1(t *testing.T, id *identity.Identity, cid, key string, value []byte, version uint64, badSig bool) []byte {
|
|
t.Helper()
|
|
var elems []byte
|
|
elems = append(elems, encText(cid)...)
|
|
elems = append(elems, encText(id.UID)...)
|
|
elems = append(elems, encText(key)...)
|
|
elems = append(elems, value...)
|
|
elems = append(elems, encUint(version)...)
|
|
elems = append(elems, encUint(oldTSsec)...)
|
|
signed := append(append([]byte(mmur1Domain), 0x86), elems...)
|
|
if badSig {
|
|
signed = append(signed, 'x')
|
|
}
|
|
sig, err := identity.Sign(id, signed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return append(append([]byte{0x87}, elems...), encBytes(sig)...)
|
|
}
|
|
|
|
func cell(t *testing.T, rec []byte, version uint64) string {
|
|
b, err := json.Marshal(map[string]any{
|
|
"record": base64.StdEncoding.EncodeToString(rec), "version": version, "ts": oldTSsec,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func importID(t *testing.T, n byte) *identity.Identity {
|
|
t.Helper()
|
|
id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return id
|
|
}
|
|
|
|
type fixture struct {
|
|
dir, oldPath, accounts string
|
|
user, stranger *identity.Identity
|
|
seekPlain, durPlain []byte
|
|
}
|
|
|
|
func buildV1(t *testing.T) fixture {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
f := fixture{dir: dir, oldPath: filepath.Join(dir, "v1.db"), accounts: filepath.Join(dir, "accounts")}
|
|
f.user, f.stranger = importID(t, 0x11), importID(t, 0x22)
|
|
if err := os.MkdirAll(f.accounts, 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := identity.SaveAccount(f.accounts, f.user); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
aead, err := identity.DeriveAEADKey(f.user)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.seekPlain = encF64(123.5)
|
|
f.durPlain = encF64(5400)
|
|
env, err := sealEnvelope(aead, f.seekPlain, envelopeAAD(cidIPFS, f.user.UID, "seek", 3), bytes.Repeat([]byte{9}, 24))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
db, err := sql.Open("sqlite", "file:"+f.oldPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
u := f.user.UID
|
|
rows := map[string]string{
|
|
"meta-core:schema-version": "2",
|
|
// record A: two cids, canonical = the dag-pb one
|
|
"file:" + ridA + "/cids/" + cidIPFS: "true",
|
|
"file:" + ridA + "/cids/" + cidSHA: "true",
|
|
"file:" + ridA + "/title": "Inception",
|
|
"file:" + ridA + "/year": "007",
|
|
"file:" + ridA + "/sizeByte": "15032385536",
|
|
"file:" + ridA + "/genres/Drama": "true",
|
|
"file:" + ridA + "/filePath": "/files/watch/Inception.mkv",
|
|
"file:" + ridA + "/mtimeNano": "1729872000000000000",
|
|
"file:" + ridA + "/cid_sha256": cidSHA,
|
|
"file:" + ridA + "/fileinfo/duration": "8878.5",
|
|
"cid:" + cidIPFS: ridA,
|
|
"cid:" + cidSHA: ridA,
|
|
// record B: its cid only in the index; claims cidSHA too (conflict)
|
|
"file:" + ridB + "/title": "Other",
|
|
"file:" + ridB + "/cids/" + cidSHA: "true",
|
|
"cid:" + cidB: ridB,
|
|
// record C: no cid at all → dropped
|
|
"file:" + ridC + "/title": "Nobody",
|
|
// orphan index row
|
|
"cid:" + cidOrph: ridD,
|
|
// scratch + derived
|
|
"btmagnet:abc": "magnet:?xt=urn:btih:abc",
|
|
"searchcov:q": "1",
|
|
"nzb:x": "y",
|
|
"idx:roots/" + ridA: "",
|
|
"dup:" + ridA + "/p": "",
|
|
"udl:idx:user:" + u + "\x1f": "",
|
|
// UDL
|
|
"udl:rec:" + u + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.user, cidIPFS, "like", encBool(true), 1, false), 1),
|
|
"udl:rec:" + u + "/" + cidIPFS + "/rating": cell(t, mmur1(t, f.user, cidIPFS, "rating", encUint(4), 1, false), 1),
|
|
"udl:rec:" + u + "/" + cidIPFS + "/seek": cell(t, mmur1(t, f.user, cidIPFS, "seek", encBytes(env), 3, false), 3),
|
|
"udl:rec:" + u + "/" + cidIPFS + "/watched": cell(t, mmur1(t, f.user, cidIPFS, "watched", encNull, 2, false), 2),
|
|
"udl:rec:" + u + "/" + cidIPFS + "/duration": cell(t, mmur1(t, f.user, cidIPFS, "duration", f.durPlain, 1, false), 1),
|
|
"udl:rec:" + u + "/self/profile:name": cell(t, mmur1(t, f.user, "self", "profile:name", encText("Alice"), 1, false), 1),
|
|
"udl:rec:" + u + "/" + cidSHA + "/like": cell(t, mmur1(t, f.user, cidSHA, "like", encBool(true), 1, true), 1),
|
|
"udl:rec:" + f.stranger.UID + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.stranger, cidIPFS, "like", encBool(false), 1, false), 1),
|
|
}
|
|
for k, v := range rows {
|
|
if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return f
|
|
}
|
|
|
|
func TestMigrate(t *testing.T) {
|
|
f := buildV1(t)
|
|
truthPath := filepath.Join(f.dir, "truth.db")
|
|
now := time.UnixMilli(1_790_000_000_000)
|
|
st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, now: now, workers: 3})
|
|
if err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
var sum bytes.Buffer
|
|
printSummary(&sum, st, false)
|
|
t.Log(sum.String())
|
|
|
|
check := func(name string, got, want int) {
|
|
t.Helper()
|
|
if got != want {
|
|
t.Errorf("%s = %d, want %d", name, got, want)
|
|
}
|
|
}
|
|
check("rids", st.Rids, 2)
|
|
check("records without cid", st.RecordsNoCID, 1)
|
|
check("orphan cid rows", st.OrphanCIDRows, 1)
|
|
check("cid conflicts", st.CIDConflicts, 1)
|
|
check("alias claims", st.AliasClaims, 1)
|
|
check("field claims", st.FieldClaims, 6) // A: title year sizeByte genres/Drama fileinfo/duration; B: title
|
|
check("refused cid_*", st.RefusedFields["cid_*"], 1)
|
|
check("watcher facts", st.WatcherFacts.total(), 2)
|
|
check("udl migrated", st.UDLMigrated.total(), 6)
|
|
check("udl no account", st.UDLDropped["no account on this box"], 1)
|
|
check("udl bad sig", st.UDLDropped["bad MMUR1 signature"], 1)
|
|
check("groups minted", st.GroupsMinted, 1) // "self"
|
|
check("cid_group rows", st.CIDGroupRows, 4)
|
|
check("scratch", st.Scratch.total(), 3)
|
|
check("principals", st.Principals, 2)
|
|
check("claims", st.ClaimsWritten, st.FieldClaims+st.AliasClaims+st.UDLMigrated.total())
|
|
|
|
// Reopen with meta-core's storage package: boot catch-up rebuilds the cache.
|
|
c := storage.NewClient()
|
|
if err := c.Connect(truthPath, filepath.Join(f.dir, "cache.db")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer c.Close()
|
|
if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := c.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for cid, want := range map[string]string{cidIPFS: ridA, cidSHA: ridA, cidB: ridB, cidOrph: ""} {
|
|
got, err := c.GetByCID(cid)
|
|
if err != nil || got != want {
|
|
t.Errorf("GetByCID(%s) = %q, %v; want %q", cid, got, err, want)
|
|
}
|
|
}
|
|
a, err := c.GetMetadataFlat(ridA)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k, want := range map[string]string{
|
|
"title": "Inception", "year": "007", "sizeByte": "15032385536", "genres/Drama": "true",
|
|
"fileinfo/duration": "8878.5", "cids/" + cidIPFS: "true", "cids/" + cidSHA: "true",
|
|
} {
|
|
if a[k] != want {
|
|
t.Errorf("record A %s = %q, want %q", k, a[k], want)
|
|
}
|
|
}
|
|
for _, k := range []string{"filePath", "mtimeNano", "cid_sha256"} {
|
|
if _, ok := a[k]; ok {
|
|
t.Errorf("record A carries %s", k)
|
|
}
|
|
}
|
|
b, err := c.GetMetadataFlat(ridB)
|
|
if err != nil || b["title"] != "Other" {
|
|
t.Errorf("record B = %v, %v", b, err)
|
|
}
|
|
ids, err := c.GetAllHashIDs()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !slices.Contains(ids, ridA) || !slices.Contains(ids, ridB) || slices.Contains(ids, ridC) || slices.Contains(ids, ridD) {
|
|
t.Errorf("roots = %v", ids)
|
|
}
|
|
if len(ids) != 3 {
|
|
t.Errorf("want 3 roots (A, B, self), got %v", ids)
|
|
}
|
|
|
|
// Legacy claims sit on the canonical cid, signed by the printed uid.
|
|
cls, err := c.ClaimsForCIDs([]string{cidIPFS}, "title")
|
|
if err != nil || len(cls) != 1 || cls[0].UID != st.LegacyUID || cls[0].TS != uint64(now.UnixMilli()) {
|
|
t.Fatalf("title claims = %+v, %v", cls, err)
|
|
}
|
|
if err := cls[0].Verify(); err != nil {
|
|
t.Error(err)
|
|
}
|
|
alias, err := c.ClaimsForCIDs([]string{cidIPFS}, "cids/"+cidSHA)
|
|
if err != nil || len(alias) != 1 || alias[0].Value.Text != "true" {
|
|
t.Errorf("alias claims = %+v, %v", alias, err)
|
|
}
|
|
|
|
// UDL claims.
|
|
u := f.user.UID
|
|
wantTS := uint64(oldTSsec) * 1000
|
|
get := func(cid, key string) claim.Claim {
|
|
t.Helper()
|
|
cl, ok, err := c.UDLGetClaim(u, cid, key)
|
|
if err != nil || !ok {
|
|
t.Fatalf("UDL %s/%s: ok=%v err=%v", cid, key, ok, err)
|
|
}
|
|
if cl.TS != wantTS {
|
|
t.Errorf("UDL %s ts = %d, want %d", key, cl.TS, wantTS)
|
|
}
|
|
if err := cl.Verify(); err != nil {
|
|
t.Errorf("UDL %s: %v", key, err)
|
|
}
|
|
return cl
|
|
}
|
|
if v := get(cidIPFS, "like").Value; v.Kind != claim.KindText || v.Text != "true" {
|
|
t.Errorf("like = %+v", v)
|
|
}
|
|
if v := get(cidIPFS, "userRating").Value; v.Kind != claim.KindText || v.Text != "4" {
|
|
t.Errorf("userRating = %+v", v)
|
|
}
|
|
if _, ok, _ := c.UDLGetClaim(u, cidIPFS, "rating"); ok {
|
|
t.Error("rating was not renamed")
|
|
}
|
|
if v := get(cidIPFS, "watched").Value; !v.IsNull() {
|
|
t.Errorf("watched tombstone = %+v", v)
|
|
}
|
|
if v := get("self", "profile:name").Value; v.Text != "Alice" {
|
|
t.Errorf("profile:name = %+v", v)
|
|
}
|
|
aead, _ := identity.DeriveAEADKey(f.user)
|
|
for key, plain := range map[string][]byte{"seek": f.seekPlain, "duration": f.durPlain} {
|
|
cl := get(cidIPFS, key)
|
|
if cl.Value.Kind != claim.KindBytes {
|
|
t.Fatalf("%s is not an envelope: %+v", key, cl.Value)
|
|
}
|
|
pt, err := openEnvelope(aead, cl.Value.Bytes, envelopeAAD(cidIPFS, u, key, cl.TS))
|
|
if err != nil || !bytes.Equal(pt, plain) {
|
|
t.Errorf("%s decrypts to %x, %v; want %x", key, pt, err, plain)
|
|
}
|
|
}
|
|
if _, ok, _ := c.UDLGetClaim(f.stranger.UID, cidIPFS, "like"); ok {
|
|
t.Error("stranger's cell migrated without an account")
|
|
}
|
|
|
|
// A second run refuses to overwrite.
|
|
if _, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts}); err == nil {
|
|
t.Error("second run overwrote the truth file")
|
|
}
|
|
}
|
|
|
|
func TestDryRunWritesNothing(t *testing.T) {
|
|
f := buildV1(t)
|
|
truthPath := filepath.Join(f.dir, "truth.db")
|
|
st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, dryRun: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := os.Stat(truthPath); !os.IsNotExist(err) {
|
|
t.Errorf("dry run created %s", truthPath)
|
|
}
|
|
if st.ClaimsWritten != st.FieldClaims+st.AliasClaims+st.UDLMigrated.total() || st.ClaimsWritten == 0 {
|
|
t.Errorf("dry-run counts: %+v", st)
|
|
}
|
|
}
|
|
|
|
func TestNoAccountsDropsUDL(t *testing.T) {
|
|
f := buildV1(t)
|
|
st, err := migrate(options{oldPath: f.oldPath, dryRun: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if st.UDLMigrated.total() != 0 || st.UDLDropped["no account on this box"] != 7 {
|
|
t.Errorf("udl: migrated %v dropped %v", st.UDLMigrated, st.UDLDropped)
|
|
}
|
|
}
|
|
|
|
func TestHalfFloat(t *testing.T) {
|
|
for h, want := range map[uint16]float64{0x3c00: 1, 0xc000: -2, 0x3e00: 1.5, 0x0000: 0, 0x7bff: 65504} {
|
|
if got := halfToFloat(h); got != want {
|
|
t.Errorf("half %04x = %v, want %v", h, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Two records listing the same cid in cids/* with no cid: index row (the same
|
|
// content stored twice): the first keeps it, a record left with no cid of its
|
|
// own folds into it, and the duplicate (cid, key) claims are skipped.
|
|
func TestSharedUnindexedCIDFolds(t *testing.T) {
|
|
dir := t.TempDir()
|
|
oldPath := filepath.Join(dir, "v1.db")
|
|
db, err := sql.Open("sqlite", "file:"+oldPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for k, v := range map[string]string{
|
|
"meta-core:schema-version": "2",
|
|
"file:" + ridA + "/cids/" + cidSHA: "true",
|
|
"file:" + ridA + "/title": "A",
|
|
"file:" + ridB + "/cids/" + cidSHA: "true",
|
|
"file:" + ridB + "/title": "B",
|
|
"file:" + ridB + "/genres/Drama": "true",
|
|
"file:" + ridC + "/cids/" + cidSHA: "true",
|
|
"file:" + ridC + "/cids/" + cidB: "true",
|
|
"file:" + ridC + "/title": "C",
|
|
} {
|
|
if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
db.Close()
|
|
truthPath := filepath.Join(dir, "truth.db")
|
|
st, err := migrate(options{oldPath: oldPath, truthPath: truthPath, workers: 2})
|
|
if err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
if st.Rids != 2 || st.RecordsFolded != 1 || st.CIDShared != 2 || st.DupClaims != 1 {
|
|
t.Errorf("rids %d folded %d shared %d dups %d", st.Rids, st.RecordsFolded, st.CIDShared, st.DupClaims)
|
|
}
|
|
out, err := sql.Open("sqlite", "file:"+truthPath+"?mode=ro")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer out.Close()
|
|
groups := map[string]string{}
|
|
rows, err := out.Query(`SELECT cid, rid FROM cid_group`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for rows.Next() {
|
|
var c, r string
|
|
if err := rows.Scan(&c, &r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
groups[c] = r
|
|
}
|
|
rows.Close()
|
|
if groups[cidSHA] != ridA || groups[cidB] != ridC || len(groups) != 2 {
|
|
t.Errorf("cid_group %v", groups)
|
|
}
|
|
var n int
|
|
if err := out.QueryRow(`SELECT count(*) FROM claim WHERE cid = ? AND key IN ('title', 'genres/Drama')`, cidSHA).Scan(&n); err != nil || n != 2 {
|
|
t.Errorf("claims on the shared cid: %d (%v)", n, err)
|
|
}
|
|
var title string
|
|
if err := out.QueryRow(`SELECT value FROM claim WHERE cid = ? AND key = 'title'`, cidB).Scan(&title); err != nil || title != "C" {
|
|
t.Errorf("record C title on its own cid: %q (%v)", title, err)
|
|
}
|
|
}
|