v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410
Build and Push Docker Image / build (push) Successful in 5m1s
Build and Push Docker Image / build (push) Successful in 5m1s
This commit is contained in:
Worph
@@ -0,0 +1,375 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/claim"
|
||||
"github.com/metazla/meta-core/internal/identity"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
cidIPFS = "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 40
|
||||
cidSHA = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 30
|
||||
cidB = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
|
||||
cidOrph = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
|
||||
ridA = "01JKR8XW5T4QMV7AHNJ2DEFGHK"
|
||||
ridB = "01JKR8XW5T4QMV7AHNJ2DEFGHM"
|
||||
ridC = "01JKR8XW5T4QMV7AHNJ2DEFGHN"
|
||||
ridD = "01JKR8XW5T4QMV7AHNJ2DEFGHP"
|
||||
oldTSsec = 1_700_000_000
|
||||
)
|
||||
|
||||
// --- minimal canonical CBOR encoder for building MMUR1 fixtures ---
|
||||
|
||||
func head(major byte, n uint64) []byte {
|
||||
m := major << 5
|
||||
switch {
|
||||
case n < 24:
|
||||
return []byte{m | byte(n)}
|
||||
case n <= 0xff:
|
||||
return []byte{m | 24, byte(n)}
|
||||
case n <= 0xffff:
|
||||
return []byte{m | 25, byte(n >> 8), byte(n)}
|
||||
case n <= 0xffffffff:
|
||||
return binary.BigEndian.AppendUint32([]byte{m | 26}, uint32(n))
|
||||
}
|
||||
return binary.BigEndian.AppendUint64([]byte{m | 27}, n)
|
||||
}
|
||||
|
||||
func encText(s string) []byte { return append(head(3, uint64(len(s))), s...) }
|
||||
func encBytes(b []byte) []byte { return append(head(2, uint64(len(b))), b...) }
|
||||
func encUint(n uint64) []byte { return head(0, n) }
|
||||
func encBool(b bool) []byte {
|
||||
if b {
|
||||
return []byte{0xf5}
|
||||
}
|
||||
return []byte{0xf4}
|
||||
}
|
||||
func encF64(f float64) []byte {
|
||||
return binary.BigEndian.AppendUint64([]byte{0xfb}, math.Float64bits(f))
|
||||
}
|
||||
|
||||
var encNull = []byte{0xf6}
|
||||
|
||||
// mmur1 builds a signed MMUR1 record; badSig signs other bytes.
|
||||
func mmur1(t *testing.T, id *identity.Identity, cid, key string, value []byte, version uint64, badSig bool) []byte {
|
||||
t.Helper()
|
||||
var elems []byte
|
||||
elems = append(elems, encText(cid)...)
|
||||
elems = append(elems, encText(id.UID)...)
|
||||
elems = append(elems, encText(key)...)
|
||||
elems = append(elems, value...)
|
||||
elems = append(elems, encUint(version)...)
|
||||
elems = append(elems, encUint(oldTSsec)...)
|
||||
signed := append(append([]byte(mmur1Domain), 0x86), elems...)
|
||||
if badSig {
|
||||
signed = append(signed, 'x')
|
||||
}
|
||||
sig, err := identity.Sign(id, signed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return append(append([]byte{0x87}, elems...), encBytes(sig)...)
|
||||
}
|
||||
|
||||
func cell(t *testing.T, rec []byte, version uint64) string {
|
||||
b, err := json.Marshal(map[string]any{
|
||||
"record": base64.StdEncoding.EncodeToString(rec), "version": version, "ts": oldTSsec,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func importID(t *testing.T, n byte) *identity.Identity {
|
||||
t.Helper()
|
||||
id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 7))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
type fixture struct {
|
||||
dir, oldPath, accounts string
|
||||
user, stranger *identity.Identity
|
||||
seekPlain, durPlain []byte
|
||||
}
|
||||
|
||||
func buildV1(t *testing.T) fixture {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
f := fixture{dir: dir, oldPath: filepath.Join(dir, "v1.db"), accounts: filepath.Join(dir, "accounts")}
|
||||
f.user, f.stranger = importID(t, 0x11), importID(t, 0x22)
|
||||
if err := os.MkdirAll(f.accounts, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := identity.SaveAccount(f.accounts, f.user); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
aead, err := identity.DeriveAEADKey(f.user)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.seekPlain = encF64(123.5)
|
||||
f.durPlain = encF64(5400)
|
||||
env, err := sealEnvelope(aead, f.seekPlain, envelopeAAD(cidIPFS, f.user.UID, "seek", 3), bytes.Repeat([]byte{9}, 24))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+f.oldPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u := f.user.UID
|
||||
rows := map[string]string{
|
||||
"meta-core:schema-version": "2",
|
||||
// record A: two cids, canonical = the dag-pb one
|
||||
"file:" + ridA + "/cids/" + cidIPFS: "true",
|
||||
"file:" + ridA + "/cids/" + cidSHA: "true",
|
||||
"file:" + ridA + "/title": "Inception",
|
||||
"file:" + ridA + "/year": "007",
|
||||
"file:" + ridA + "/sizeByte": "15032385536",
|
||||
"file:" + ridA + "/genres/Drama": "true",
|
||||
"file:" + ridA + "/filePath": "/files/watch/Inception.mkv",
|
||||
"file:" + ridA + "/mtimeNano": "1729872000000000000",
|
||||
"file:" + ridA + "/cid_sha256": cidSHA,
|
||||
"file:" + ridA + "/fileinfo/duration": "8878.5",
|
||||
"cid:" + cidIPFS: ridA,
|
||||
"cid:" + cidSHA: ridA,
|
||||
// record B: its cid only in the index; claims cidSHA too (conflict)
|
||||
"file:" + ridB + "/title": "Other",
|
||||
"file:" + ridB + "/cids/" + cidSHA: "true",
|
||||
"cid:" + cidB: ridB,
|
||||
// record C: no cid at all → dropped
|
||||
"file:" + ridC + "/title": "Nobody",
|
||||
// orphan index row
|
||||
"cid:" + cidOrph: ridD,
|
||||
// scratch + derived
|
||||
"btmagnet:abc": "magnet:?xt=urn:btih:abc",
|
||||
"searchcov:q": "1",
|
||||
"nzb:x": "y",
|
||||
"idx:roots/" + ridA: "",
|
||||
"dup:" + ridA + "/p": "",
|
||||
"udl:idx:user:" + u + "\x1f": "",
|
||||
// UDL
|
||||
"udl:rec:" + u + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.user, cidIPFS, "like", encBool(true), 1, false), 1),
|
||||
"udl:rec:" + u + "/" + cidIPFS + "/rating": cell(t, mmur1(t, f.user, cidIPFS, "rating", encUint(4), 1, false), 1),
|
||||
"udl:rec:" + u + "/" + cidIPFS + "/seek": cell(t, mmur1(t, f.user, cidIPFS, "seek", encBytes(env), 3, false), 3),
|
||||
"udl:rec:" + u + "/" + cidIPFS + "/watched": cell(t, mmur1(t, f.user, cidIPFS, "watched", encNull, 2, false), 2),
|
||||
"udl:rec:" + u + "/" + cidIPFS + "/duration": cell(t, mmur1(t, f.user, cidIPFS, "duration", f.durPlain, 1, false), 1),
|
||||
"udl:rec:" + u + "/self/profile:name": cell(t, mmur1(t, f.user, "self", "profile:name", encText("Alice"), 1, false), 1),
|
||||
"udl:rec:" + u + "/" + cidSHA + "/like": cell(t, mmur1(t, f.user, cidSHA, "like", encBool(true), 1, true), 1),
|
||||
"udl:rec:" + f.stranger.UID + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.stranger, cidIPFS, "like", encBool(false), 1, false), 1),
|
||||
}
|
||||
for k, v := range rows {
|
||||
if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
f := buildV1(t)
|
||||
truthPath := filepath.Join(f.dir, "truth.db")
|
||||
now := time.UnixMilli(1_790_000_000_000)
|
||||
st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, now: now, workers: 3})
|
||||
if err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
var sum bytes.Buffer
|
||||
printSummary(&sum, st, false)
|
||||
t.Log(sum.String())
|
||||
|
||||
check := func(name string, got, want int) {
|
||||
t.Helper()
|
||||
if got != want {
|
||||
t.Errorf("%s = %d, want %d", name, got, want)
|
||||
}
|
||||
}
|
||||
check("rids", st.Rids, 2)
|
||||
check("records without cid", st.RecordsNoCID, 1)
|
||||
check("orphan cid rows", st.OrphanCIDRows, 1)
|
||||
check("cid conflicts", st.CIDConflicts, 1)
|
||||
check("alias claims", st.AliasClaims, 1)
|
||||
check("field claims", st.FieldClaims, 6) // A: title year sizeByte genres/Drama fileinfo/duration; B: title
|
||||
check("refused cid_*", st.RefusedFields["cid_*"], 1)
|
||||
check("watcher facts", st.WatcherFacts.total(), 2)
|
||||
check("udl migrated", st.UDLMigrated.total(), 6)
|
||||
check("udl no account", st.UDLDropped["no account on this box"], 1)
|
||||
check("udl bad sig", st.UDLDropped["bad MMUR1 signature"], 1)
|
||||
check("groups minted", st.GroupsMinted, 1) // "self"
|
||||
check("cid_group rows", st.CIDGroupRows, 4)
|
||||
check("scratch", st.Scratch.total(), 3)
|
||||
check("principals", st.Principals, 2)
|
||||
check("claims", st.ClaimsWritten, st.FieldClaims+st.AliasClaims+st.UDLMigrated.total())
|
||||
|
||||
// Reopen with meta-core's storage package: boot catch-up rebuilds the cache.
|
||||
c := storage.NewClient()
|
||||
if err := c.Connect(truthPath, filepath.Join(f.dir, "cache.db")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer c.Close()
|
||||
if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for cid, want := range map[string]string{cidIPFS: ridA, cidSHA: ridA, cidB: ridB, cidOrph: ""} {
|
||||
got, err := c.GetByCID(cid)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("GetByCID(%s) = %q, %v; want %q", cid, got, err, want)
|
||||
}
|
||||
}
|
||||
a, err := c.GetMetadataFlat(ridA)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for k, want := range map[string]string{
|
||||
"title": "Inception", "year": "007", "sizeByte": "15032385536", "genres/Drama": "true",
|
||||
"fileinfo/duration": "8878.5", "cids/" + cidIPFS: "true", "cids/" + cidSHA: "true",
|
||||
} {
|
||||
if a[k] != want {
|
||||
t.Errorf("record A %s = %q, want %q", k, a[k], want)
|
||||
}
|
||||
}
|
||||
for _, k := range []string{"filePath", "mtimeNano", "cid_sha256"} {
|
||||
if _, ok := a[k]; ok {
|
||||
t.Errorf("record A carries %s", k)
|
||||
}
|
||||
}
|
||||
b, err := c.GetMetadataFlat(ridB)
|
||||
if err != nil || b["title"] != "Other" {
|
||||
t.Errorf("record B = %v, %v", b, err)
|
||||
}
|
||||
ids, err := c.GetAllHashIDs()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(ids, ridA) || !slices.Contains(ids, ridB) || slices.Contains(ids, ridC) || slices.Contains(ids, ridD) {
|
||||
t.Errorf("roots = %v", ids)
|
||||
}
|
||||
if len(ids) != 3 {
|
||||
t.Errorf("want 3 roots (A, B, self), got %v", ids)
|
||||
}
|
||||
|
||||
// Legacy claims sit on the canonical cid, signed by the printed uid.
|
||||
cls, err := c.ClaimsForCIDs([]string{cidIPFS}, "title")
|
||||
if err != nil || len(cls) != 1 || cls[0].UID != st.LegacyUID || cls[0].TS != uint64(now.UnixMilli()) {
|
||||
t.Fatalf("title claims = %+v, %v", cls, err)
|
||||
}
|
||||
if err := cls[0].Verify(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
alias, err := c.ClaimsForCIDs([]string{cidIPFS}, "cids/"+cidSHA)
|
||||
if err != nil || len(alias) != 1 || alias[0].Value.Text != "true" {
|
||||
t.Errorf("alias claims = %+v, %v", alias, err)
|
||||
}
|
||||
|
||||
// UDL claims.
|
||||
u := f.user.UID
|
||||
wantTS := uint64(oldTSsec) * 1000
|
||||
get := func(cid, key string) claim.Claim {
|
||||
t.Helper()
|
||||
cl, ok, err := c.UDLGetClaim(u, cid, key)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("UDL %s/%s: ok=%v err=%v", cid, key, ok, err)
|
||||
}
|
||||
if cl.TS != wantTS {
|
||||
t.Errorf("UDL %s ts = %d, want %d", key, cl.TS, wantTS)
|
||||
}
|
||||
if err := cl.Verify(); err != nil {
|
||||
t.Errorf("UDL %s: %v", key, err)
|
||||
}
|
||||
return cl
|
||||
}
|
||||
if v := get(cidIPFS, "like").Value; v.Kind != claim.KindText || v.Text != "true" {
|
||||
t.Errorf("like = %+v", v)
|
||||
}
|
||||
if v := get(cidIPFS, "userRating").Value; v.Kind != claim.KindText || v.Text != "4" {
|
||||
t.Errorf("userRating = %+v", v)
|
||||
}
|
||||
if _, ok, _ := c.UDLGetClaim(u, cidIPFS, "rating"); ok {
|
||||
t.Error("rating was not renamed")
|
||||
}
|
||||
if v := get(cidIPFS, "watched").Value; !v.IsNull() {
|
||||
t.Errorf("watched tombstone = %+v", v)
|
||||
}
|
||||
if v := get("self", "profile:name").Value; v.Text != "Alice" {
|
||||
t.Errorf("profile:name = %+v", v)
|
||||
}
|
||||
aead, _ := identity.DeriveAEADKey(f.user)
|
||||
for key, plain := range map[string][]byte{"seek": f.seekPlain, "duration": f.durPlain} {
|
||||
cl := get(cidIPFS, key)
|
||||
if cl.Value.Kind != claim.KindBytes {
|
||||
t.Fatalf("%s is not an envelope: %+v", key, cl.Value)
|
||||
}
|
||||
pt, err := openEnvelope(aead, cl.Value.Bytes, envelopeAAD(cidIPFS, u, key, cl.TS))
|
||||
if err != nil || !bytes.Equal(pt, plain) {
|
||||
t.Errorf("%s decrypts to %x, %v; want %x", key, pt, err, plain)
|
||||
}
|
||||
}
|
||||
if _, ok, _ := c.UDLGetClaim(f.stranger.UID, cidIPFS, "like"); ok {
|
||||
t.Error("stranger's cell migrated without an account")
|
||||
}
|
||||
|
||||
// A second run refuses to overwrite.
|
||||
if _, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts}); err == nil {
|
||||
t.Error("second run overwrote the truth file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDryRunWritesNothing(t *testing.T) {
|
||||
f := buildV1(t)
|
||||
truthPath := filepath.Join(f.dir, "truth.db")
|
||||
st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, dryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(truthPath); !os.IsNotExist(err) {
|
||||
t.Errorf("dry run created %s", truthPath)
|
||||
}
|
||||
if st.ClaimsWritten != st.FieldClaims+st.AliasClaims+st.UDLMigrated.total() || st.ClaimsWritten == 0 {
|
||||
t.Errorf("dry-run counts: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoAccountsDropsUDL(t *testing.T) {
|
||||
f := buildV1(t)
|
||||
st, err := migrate(options{oldPath: f.oldPath, dryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.UDLMigrated.total() != 0 || st.UDLDropped["no account on this box"] != 7 {
|
||||
t.Errorf("udl: migrated %v dropped %v", st.UDLMigrated, st.UDLDropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHalfFloat(t *testing.T) {
|
||||
for h, want := range map[uint16]float64{0x3c00: 1, 0xc000: -2, 0x3e00: 1.5, 0x0000: 0, 0x7bff: 65504} {
|
||||
if got := halfToFloat(h); got != want {
|
||||
t.Errorf("half %04x = %v, want %v", h, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user