v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410

tools/claims-migrate/main.go

@@ -0,0 +1,71 @@
+// Command claims-migrate is the one-off conversion of a meta-core v1.2.0 store
+// (one kv table, schema version 2) into a v2.0.0 truth database (the claims
+// model, schema version 3) — docs/implementation/metacore-claims-model.md §11.
+// It is run once by hand on watch.nsl.sh with meta-core stopped, and deleted
+// afterwards; nothing in the build references it.
+//
+//	claims-migrate -old /meta-core/db/meta-core.db -truth /out/meta-core.db \
+//	               -accounts /meta-core/identity/accounts [-dry-run]
+//
+// What it does:
+//   - generates a throwaway "legacy" secp256k1 key in memory (its uid is
+//     printed; the private key is never written anywhere);
+//   - every record (rid) with at least one cid: each file:<rid>/<field> becomes
+//     a claim on the record's canonical cid (internal/cid rank) signed by the
+//     legacy key at one conversion ts; cids/<other> members become alias claims
+//     (canonical, cids/<other>) = "true"; filePath / mtimeNano (watcher facts,
+//     refilled into local_file) and keys meta-core refuses (cid_*, midhash256,
+//     canonical_cid, __fields__) are skipped; records with no cid are dropped;
+//   - cid_group from the cid:<cid> → rid index (rids keep their UUIDs);
+//   - UDL cells (MMUR1 records, signature checked) become MMUR2 claims signed
+//     by the account key found in -accounts; rating → userRating; private-tier
+//     envelopes are decrypted (AAD bound to the old version) and re-encrypted
+//     with a fresh nonce and the MMUR2 AAD (bound to ts); cells whose account is
+//     not on the box are dropped;
+//   - btmagnet:* / searchcov:* / other service scratch keys are dropped (they
+//     live in the cache now and are refetched); derived bookkeeping is skipped.
+//
+// Every claim is checked with claim.Verify before it is stored. The output is
+// created with meta-core's own storage code, so the truth schema cannot drift;
+// meta-core rebuilds the cache from it on first boot.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+)
+
+func main() {
+	var o options
+	flag.StringVar(&o.oldPath, "old", "", "path to the v1 meta-core.db (opened read-only; copy it with its -wal/-shm first)")
+	flag.StringVar(&o.truthPath, "truth", "", "output truth database path (must not exist)")
+	flag.StringVar(&o.accountsDir, "accounts", "", "identity accounts dir ({META_CORE_PATH}/identity/accounts); empty = drop every UDL cell")
+	flag.BoolVar(&o.dryRun, "dry-run", false, "convert, sign and verify everything but write nothing; print counts")
+	flag.IntVar(&o.workers, "workers", runtime.NumCPU(), "signing goroutines")
+	flag.Parse()
+	if o.oldPath == "" || (o.truthPath == "" && !o.dryRun) {
+		fmt.Fprintln(os.Stderr, "usage: claims-migrate -old <v1 meta-core.db> -truth <new truth db> [-accounts <dir>] [-dry-run]")
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+	if o.accountsDir != "" {
+		if fi, err := os.Stat(o.accountsDir); err != nil || !fi.IsDir() {
+			fmt.Fprintf(os.Stderr, "accounts dir %s is not a directory\n", o.accountsDir)
+			os.Exit(2)
+		}
+	} else {
+		fmt.Fprintln(os.Stderr, "warning: no -accounts: every UDL cell will be dropped")
+	}
+	o.log = os.Stderr
+	st, err := migrate(o)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "claims-migrate: %v\n", err)
+		os.Exit(1)
+	}
+	printSummary(os.Stdout, st, o.dryRun)
+	if !o.dryRun {
+		fmt.Printf("\ntruth database written: %s\n", o.truthPath)
+	}
+}

tools/claims-migrate/migrate.go

@@ -0,0 +1,902 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"database/sql"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	_ "modernc.org/sqlite"
+
+	"github.com/metazla/meta-core/internal/cid"
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+type options struct {
+	oldPath     string
+	truthPath   string
+	accountsDir string
+	dryRun      bool
+	now         time.Time // conversion timestamp; zero = time.Now()
+	workers     int
+	log         io.Writer
+}
+
+// counts is a labelled tally, printed sorted.
+type counts map[string]int
+
+func (c counts) total() int {
+	n := 0
+	for _, v := range c {
+		n += v
+	}
+	return n
+}
+
+type stats struct {
+	LegacyUID string
+	ConvTS    uint64
+
+	Rids          int // records migrated (have at least one cid)
+	RidsNoClaims  int // migrated records whose only rows were cids / watcher facts
+	RecordsNoCID  int // dropped: no cid at all
+	FieldClaims   int
+	AliasClaims   int
+	WatcherFacts  counts // filePath / mtimeNano skipped
+	RefusedFields counts // by reason
+	Malformed     int    // file: keys without <rid>/<field>
+
+	CIDGroupRows  int
+	CIDConflicts  int // cids/<c> on a rid while cid:<c> names another rid
+	OrphanCIDRows int // cid:<c> → rid with no file: rows
+
+	UDLMigrated  counts
+	UDLDropped   counts
+	UDLSigners   int
+	GroupsMinted int // cid_group rows minted for UDL cids not in any record
+
+	Scratch counts // service scratch keys dropped (btmagnet, searchcov, …)
+	Derived counts // derived bookkeeping skipped (idx, ev, dup, udl:idx, …)
+
+	Principals    int
+	ClaimsWritten int
+	Elapsed       time.Duration
+}
+
+func newStats() *stats {
+	return &stats{
+		WatcherFacts: counts{}, RefusedFields: counts{},
+		UDLMigrated: counts{}, UDLDropped: counts{},
+		Scratch: counts{}, Derived: counts{},
+	}
+}
+
+// job is one claim to sign with id.
+type job struct {
+	cl claim.Claim
+	id *identity.Identity
+}
+
+const batchSize = 4096
+
+func (o *options) logf(format string, args ...any) {
+	if o.log != nil {
+		fmt.Fprintf(o.log, format+"\n", args...)
+	}
+}
+
+// migrate converts a meta-core v1.2.0 kv store into a v2.0.0 truth database.
+func migrate(o options) (st *stats, err error) {
+	started := time.Now()
+	st = newStats()
+	if o.workers <= 0 {
+		o.workers = runtime.NumCPU()
+	}
+	if o.now.IsZero() {
+		o.now = time.Now()
+	}
+	st.ConvTS = uint64(o.now.UnixMilli())
+
+	old, err := openOld(o.oldPath)
+	if err != nil {
+		return nil, err
+	}
+	defer old.Close()
+
+	legacy, err := identity.Generate() // in memory only, never saved
+	if err != nil {
+		return nil, err
+	}
+	st.LegacyUID = legacy.UID
+	o.logf("legacy signer uid: %s (private key kept in memory only)", legacy.UID)
+
+	var out *sql.DB
+	if !o.dryRun {
+		if out, err = createTruth(o.truthPath); err != nil {
+			return nil, err
+		}
+		defer func() {
+			if out != nil {
+				out.Close()
+			}
+			if err != nil {
+				removeDB(o.truthPath)
+			}
+		}()
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Pipeline: producer (this goroutine) → signers → one writer.
+	jobs := make(chan []job, o.workers*2)
+	signed := make(chan []claim.Claim, o.workers*2)
+	var firstErr error
+	var errMu sync.Mutex
+	fail := func(e error) {
+		errMu.Lock()
+		if firstErr == nil {
+			firstErr = e
+		}
+		errMu.Unlock()
+		cancel()
+	}
+	var wg sync.WaitGroup
+	for range o.workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for batch := range jobs {
+				res := make([]claim.Claim, len(batch))
+				for i := range batch {
+					cl := batch[i].cl
+					if e := cl.Sign(batch[i].id); e != nil {
+						fail(fmt.Errorf("sign %s/%s: %w", cl.CID, cl.Key, e))
+						return
+					}
+					if e := cl.Verify(); e != nil {
+						fail(fmt.Errorf("verify %s/%s: %w", cl.CID, cl.Key, e))
+						return
+					}
+					res[i] = cl
+				}
+				select {
+				case signed <- res:
+				case <-ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+	go func() { wg.Wait(); close(signed) }()
+
+	w := &writer{db: out, pids: map[string]int64{}}
+	writerDone := make(chan error, 1)
+	go func() {
+		var e error
+		for batch := range signed {
+			if e == nil {
+				if e = w.write(batch); e != nil {
+					fail(e)
+				}
+			}
+		}
+		writerDone <- e
+	}()
+
+	var pending []job
+	send := func(j job) error {
+		pending = append(pending, j)
+		if len(pending) < batchSize {
+			return nil
+		}
+		b := pending
+		pending = nil
+		select {
+		case jobs <- b:
+			return nil
+		case <-ctx.Done():
+			return errors.New("aborted")
+		}
+	}
+
+	groups, prodErr := produce(ctx, old, o, st, legacy, send)
+	if prodErr == nil && len(pending) > 0 {
+		select {
+		case jobs <- pending:
+		case <-ctx.Done():
+			prodErr = errors.New("aborted")
+		}
+	}
+	close(jobs)
+	wErr := <-writerDone
+	errMu.Lock()
+	fe := firstErr
+	errMu.Unlock()
+	if fe != nil {
+		return nil, fe
+	}
+	if prodErr != nil {
+		return nil, prodErr
+	}
+	if wErr != nil {
+		return nil, wErr
+	}
+
+	st.CIDGroupRows = len(groups)
+	st.Principals = len(w.pids)
+	st.ClaimsWritten = int(w.seq)
+	if !o.dryRun {
+		if err = w.finish(groups); err != nil {
+			return nil, err
+		}
+		if _, err = out.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+			return nil, err
+		}
+		if err = out.Close(); err != nil {
+			out = nil
+			return nil, err
+		}
+		out = nil
+		if err = checkTruth(o.truthPath); err != nil {
+			return nil, fmt.Errorf("re-opening the result with meta-core's storage: %w", err)
+		}
+	}
+	st.Elapsed = time.Since(started)
+	return st, nil
+}
+
+// produce reads the old store and emits every claim; it returns cid_group.
+func produce(ctx context.Context, old *sql.DB, o options, st *stats, legacy *identity.Identity, send func(job) error) (map[string]string, error) {
+	// 1. cid index.
+	index := map[string]string{}
+	ridIdx := map[string][]string{}
+	rows, err := old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'cid:' AND k < 'cid;'`)
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var k, v string
+		if err := rows.Scan(&k, &v); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		c := k[len("cid:"):]
+		if c == "" || v == "" {
+			continue
+		}
+		index[c] = v
+		ridIdx[v] = append(ridIdx[v], c)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	o.logf("cid index: %d rows", len(index))
+
+	groups := map[string]string{}
+
+	// 2. records, one rid at a time (the file: range is contiguous per rid).
+	rows, err = old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'file:' AND k < 'file;' ORDER BY k`)
+	if err != nil {
+		return nil, err
+	}
+	seen := map[string]bool{}
+	cur := ""
+	var fields [][2]string
+	flush := func() error {
+		if cur == "" {
+			return nil
+		}
+		if seen[cur] {
+			return fmt.Errorf("file: range for %s is not contiguous", cur)
+		}
+		seen[cur] = true
+		err := emitRecord(o, st, legacy, index, ridIdx, groups, cur, fields, send)
+		fields = fields[:0]
+		return err
+	}
+	n := 0
+	for rows.Next() {
+		var k, v string
+		if err := rows.Scan(&k, &v); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		rid, field, ok := strings.Cut(k[len("file:"):], "/")
+		if !ok || rid == "" || field == "" {
+			st.Malformed++
+			continue
+		}
+		if rid != cur {
+			if err := flush(); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			cur = rid
+		}
+		fields = append(fields, [2]string{field, v})
+		if n++; n%1_000_000 == 0 {
+			o.logf("  … %d file: rows, %d records", n, len(seen))
+		}
+	}
+	if err := rows.Err(); err != nil {
+		rows.Close()
+		return nil, err
+	}
+	rows.Close()
+	if err := flush(); err != nil {
+		return nil, err
+	}
+	for c := range index {
+		if _, ok := groups[c]; !ok {
+			st.OrphanCIDRows++
+		}
+	}
+	o.logf("records: %d migrated, %d without a cid dropped", st.Rids, st.RecordsNoCID)
+
+	// 3. UDL cells.
+	if err := emitUDL(ctx, old, o, st, groups, send); err != nil {
+		return nil, err
+	}
+
+	// 4. everything else: count and drop.
+	rows, err = old.QueryContext(ctx, `SELECT k FROM kv
+		WHERE NOT (k >= 'file:' AND k < 'file;') AND NOT (k >= 'cid:' AND k < 'cid;')
+		  AND NOT (k >= 'udl:rec:' AND k < 'udl:rec;')`)
+	if err != nil {
+		return nil, err
+	}
+	for rows.Next() {
+		var k string
+		if err := rows.Scan(&k); err != nil {
+			rows.Close()
+			return nil, err
+		}
+		classifyOther(st, k)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	return groups, nil
+}
+
+func classifyOther(st *stats, k string) {
+	for _, p := range []string{"idx:", "ev:", "evhwm:", "dup:", "udl:idx:", "meta-core:", "meta:"} {
+		if strings.HasPrefix(k, p) {
+			st.Derived[p+"*"]++
+			return
+		}
+	}
+	if p, _, ok := strings.Cut(k, ":"); ok && p != "" {
+		st.Scratch[p+":*"]++
+		return
+	}
+	st.Scratch["(no prefix)"]++
+}
+
+// refusedReason mirrors storage.claimKeyAllowed + the write path's length
+// checks: keys meta-core's POST /api/claims would refuse.
+func refusedReason(key string) string {
+	switch {
+	case strings.HasPrefix(key, "cid_"):
+		return "cid_*"
+	case key == "midhash256", key == "canonical_cid", key == "__fields__":
+		return key
+	case strings.HasPrefix(key, "/"), strings.Contains(key, "\x00"):
+		return "malformed key"
+	case len(key) > 4096:
+		return "key too long"
+	}
+	return ""
+}
+
+func emitRecord(o options, st *stats, legacy *identity.Identity, index map[string]string,
+	ridIdx map[string][]string, groups map[string]string, rid string, fields [][2]string, send func(job) error) error {
+	// Members: cids/* the cid index does not assign elsewhere, plus every cid
+	// the index assigns here.
+	member := map[string]bool{}
+	var members []string
+	for _, f := range fields {
+		c, ok := strings.CutPrefix(f[0], storage.CIDsKeyPrefix)
+		if !ok || c == "" {
+			continue
+		}
+		if r, ok := index[c]; ok && r != rid {
+			st.CIDConflicts++
+			continue
+		}
+		if !member[c] {
+			member[c] = true
+			members = append(members, c)
+		}
+	}
+	for _, c := range ridIdx[rid] {
+		if !member[c] {
+			member[c] = true
+			members = append(members, c)
+		}
+	}
+	var usable []string
+	for _, c := range members {
+		if len(c) <= 512 {
+			usable = append(usable, c)
+		}
+	}
+	if len(usable) == 0 {
+		st.RecordsNoCID++
+		return nil
+	}
+	canonical := usable[0]
+	for _, c := range usable[1:] {
+		canonical = cid.Better(canonical, c)
+	}
+	for _, c := range usable {
+		groups[c] = rid
+	}
+	st.Rids++
+	emitted := 0
+	mk := func(key, value string) error {
+		emitted++
+		return send(job{id: legacy, cl: claim.Claim{
+			CID: canonical, UID: legacy.UID, Key: key, Value: claim.Text(value), TS: st.ConvTS,
+		}})
+	}
+	for _, f := range fields {
+		key, val := f[0], f[1]
+		if c, ok := strings.CutPrefix(key, storage.CIDsKeyPrefix); ok {
+			if c != canonical && member[c] && len(c) <= 512 && groups[c] == rid {
+				st.AliasClaims++
+				if err := mk(key, "true"); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+		if key == "filePath" || key == "mtimeNano" {
+			st.WatcherFacts[key]++
+			continue
+		}
+		if r := refusedReason(key); r != "" {
+			st.RefusedFields[r]++
+			continue
+		}
+		st.FieldClaims++
+		if err := mk(key, val); err != nil {
+			return err
+		}
+	}
+	if emitted == 0 {
+		st.RidsNoClaims++
+	}
+	return nil
+}
+
+// publicUDLKeys are the plaintext tier (meta-watch record/tiers.rs); every
+// other key is private and travels as an envelope.
+var publicUDLKeys = map[string]bool{"like": true, "userRating": true, "profile:name": true, "profile:avatar": true}
+
+type udlCellJSON struct {
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
+
+func decodeB64(s string) ([]byte, error) {
+	for _, enc := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding, base64.URLEncoding, base64.RawURLEncoding} {
+		if b, err := enc.DecodeString(s); err == nil {
+			return b, nil
+		}
+	}
+	return nil, errors.New("record is not base64")
+}
+
+// udlTS converts an MMUR1 ts (seconds; tolerate ms) to MMUR2 milliseconds.
+func udlTS(ts int64, fallback uint64) uint64 {
+	switch {
+	case ts <= 0:
+		return fallback
+	case ts < 100_000_000_000: // seconds (before year 5138)
+		return uint64(ts) * 1000
+	default:
+		return uint64(ts)
+	}
+}
+
+type account struct {
+	id      *identity.Identity
+	aeadKey []byte
+}
+
+func emitUDL(ctx context.Context, old *sql.DB, o options, st *stats, groups map[string]string, send func(job) error) error {
+	accounts := map[string]*account{}
+	loadAccount := func(uid string) (*account, error) {
+		if a, ok := accounts[uid]; ok {
+			return a, nil
+		}
+		var a *account
+		if o.accountsDir != "" && identity.ValidUID(uid) {
+			id, err := identity.LoadByUID(o.accountsDir, uid)
+			if err != nil {
+				return nil, fmt.Errorf("account %s: %w", uid, err)
+			}
+			if id != nil {
+				if id.UID != uid {
+					return nil, fmt.Errorf("account file %s.json holds uid %s", uid, id.UID)
+				}
+				k, err := identity.DeriveAEADKey(id)
+				if err != nil {
+					return nil, err
+				}
+				a = &account{id: id, aeadKey: k}
+			}
+		}
+		accounts[uid] = a
+		return a, nil
+	}
+
+	type cellKey struct{ uid, cid, key string }
+	best := map[cellKey]job{}
+	var order []cellKey
+	drop := func(reason string) { st.UDLDropped[reason]++ }
+	kinds := map[cellKey]string{}
+
+	rows, err := old.QueryContext(ctx, `SELECT k, v FROM kv WHERE k >= 'udl:rec:' AND k < 'udl:rec;' ORDER BY k`)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var k, v string
+		if err := rows.Scan(&k, &v); err != nil {
+			return err
+		}
+		parts := strings.SplitN(k[len("udl:rec:"):], "/", 3)
+		if len(parts) != 3 {
+			drop("malformed cell key")
+			continue
+		}
+		var cell udlCellJSON
+		if err := json.Unmarshal([]byte(v), &cell); err != nil || cell.Record == "" {
+			drop("malformed cell JSON")
+			continue
+		}
+		raw, err := decodeB64(cell.Record)
+		if err != nil {
+			drop("record not base64")
+			continue
+		}
+		rec, err := decodeMMUR1(raw)
+		if err != nil {
+			drop("record not MMUR1 CBOR")
+			continue
+		}
+		if rec.UID != parts[0] || rec.CID != parts[1] || rec.Key != parts[2] {
+			drop("record does not match its cell")
+			continue
+		}
+		if !identity.ValidUID(rec.UID) || identity.Verify(rec.UID, rec.Signed, rec.Sig) != nil {
+			drop("bad MMUR1 signature")
+			continue
+		}
+		acct, err := loadAccount(rec.UID)
+		if err != nil {
+			return err
+		}
+		if acct == nil {
+			drop("no account on this box")
+			continue
+		}
+		key := rec.Key
+		if key == "rating" {
+			key = "userRating" // §14: a user's rating is userRating
+		}
+		if r := refusedReason(key); r != "" || rec.CID == "" || len(rec.CID) > 512 {
+			drop("key or cid refused by meta-core")
+			continue
+		}
+		ts := udlTS(rec.TS, st.ConvTS)
+		cl := claim.Claim{CID: rec.CID, UID: rec.UID, Key: key, TS: ts}
+		kind := ""
+		switch {
+		case rec.Value.kind == cNull:
+			kind = "tombstone"
+		case publicUDLKeys[key]:
+			t, ok := rec.Value.wireText()
+			if !ok {
+				drop("public key holds bytes")
+				continue
+			}
+			cl.Value, kind = claim.Text(t), "public"
+		default:
+			var plain []byte
+			if rec.Value.kind == cBytes {
+				plain, err = openEnvelope(acct.aeadKey, rec.Value.bytes, envelopeAAD(rec.CID, rec.UID, rec.Key, rec.Version))
+				if err != nil {
+					drop("private envelope does not decrypt")
+					continue
+				}
+				kind = "private (re-encrypted)"
+			} else {
+				// A private-tier key stored in plaintext: encrypt it now, its
+				// plaintext being the CBOR of the typed value, as crypto.rs.
+				plain = append([]byte(nil), rec.Value.raw...)
+				kind = "private (plaintext, now encrypted)"
+			}
+			nonce := make([]byte, 24)
+			if _, err := rand.Read(nonce); err != nil {
+				return err
+			}
+			env, err := sealEnvelope(acct.aeadKey, plain, envelopeAAD(rec.CID, rec.UID, key, ts), nonce)
+			if err != nil {
+				return err
+			}
+			cl.Value = claim.Bytes(env)
+		}
+		ck := cellKey{rec.UID, rec.CID, key}
+		if prev, ok := best[ck]; ok {
+			drop("superseded (rating + userRating)")
+			if prev.cl.TS >= ts {
+				continue
+			}
+		} else {
+			order = append(order, ck)
+		}
+		best[ck], kinds[ck] = job{cl: cl, id: acct.id}, kind
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	signers := map[string]bool{}
+	for _, ck := range order {
+		j := best[ck]
+		if _, ok := groups[j.cl.CID]; !ok {
+			rid, err := storage.NewUUID()
+			if err != nil {
+				return err
+			}
+			groups[j.cl.CID] = rid
+			st.GroupsMinted++
+		}
+		st.UDLMigrated[kinds[ck]]++
+		signers[j.cl.UID] = true
+		if err := send(j); err != nil {
+			return err
+		}
+	}
+	st.UDLSigners = len(signers)
+	return nil
+}
+
+// writer stores signed claims in arrival order (seq = arrival order).
+type writer struct {
+	db   *sql.DB
+	pids map[string]int64
+	seq  int64
+}
+
+func (w *writer) write(batch []claim.Claim) error {
+	if w.db == nil { // dry run
+		for _, cl := range batch {
+			if _, ok := w.pids[cl.UID]; !ok {
+				w.pids[cl.UID] = int64(len(w.pids) + 1)
+			}
+		}
+		w.seq += int64(len(batch))
+		return nil
+	}
+	tx, err := w.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	ins, err := tx.Prepare(`INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`)
+	if err != nil {
+		return err
+	}
+	defer ins.Close()
+	for _, cl := range batch {
+		pid, ok := w.pids[cl.UID]
+		if !ok {
+			if err := tx.QueryRow(`INSERT INTO principal(uid) VALUES(?) RETURNING pid`, cl.UID).Scan(&pid); err != nil {
+				return fmt.Errorf("principal %s: %w", cl.UID, err)
+			}
+			w.pids[cl.UID] = pid
+		}
+		w.seq++
+		if _, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq); err != nil {
+			return fmt.Errorf("claim (%s, %s, %s): %w", cl.CID, cl.Key, cl.UID, err)
+		}
+	}
+	return tx.Commit()
+}
+
+// finish writes cid_group and last_seq.
+func (w *writer) finish(groups map[string]string) error {
+	tx, err := w.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	ins, err := tx.Prepare(`INSERT INTO cid_group(cid, rid) VALUES(?, ?)`)
+	if err != nil {
+		return err
+	}
+	defer ins.Close()
+	keys := make([]string, 0, len(groups))
+	for c := range groups {
+		keys = append(keys, c)
+	}
+	sort.Strings(keys)
+	for _, c := range keys {
+		if _, err := ins.Exec(c, groups[c]); err != nil {
+			return fmt.Errorf("cid_group %s: %w", c, err)
+		}
+	}
+	if _, err := tx.Exec(`INSERT INTO meta(name, value) VALUES('last_seq', ?)
+		ON CONFLICT(name) DO UPDATE SET value = excluded.value`, fmt.Sprint(w.seq)); err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+
+// valueArg is storage's column encoding: TEXT, BLOB or NULL (STRICT ANY keeps it).
+func valueArg(v claim.Value) any {
+	switch v.Kind {
+	case claim.KindText:
+		return v.Text
+	case claim.KindBytes:
+		if v.Bytes == nil {
+			return []byte{}
+		}
+		return v.Bytes
+	}
+	return nil
+}
+
+func openOld(path string) (*sql.DB, error) {
+	if _, err := os.Stat(path); err != nil {
+		return nil, fmt.Errorf("old store: %w", err)
+	}
+	db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=query_only(1)&_pragma=busy_timeout(10000)")
+	if err != nil {
+		return nil, err
+	}
+	var name string
+	if err := db.QueryRow(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'kv'`).Scan(&name); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("%s has no kv table — not a meta-core v1.x store: %w", path, err)
+	}
+	var ver string
+	err = db.QueryRow(`SELECT v FROM kv WHERE k = 'meta-core:schema-version'`).Scan(&ver)
+	switch {
+	case errors.Is(err, sql.ErrNoRows):
+		// pre-sentinel store: accepted, the key layout is the same
+	case err != nil:
+		db.Close()
+		return nil, err
+	case ver != "2":
+		db.Close()
+		return nil, fmt.Errorf("%s has schema version %q, this tool converts version \"2\"", path, ver)
+	}
+	return db, nil
+}
+
+// createTruth creates the truth file with meta-core's own schema code
+// (storage.Client.Connect + EnsureSchemaVersion stamp schema-version 3), then
+// reopens it for the bulk load.
+func createTruth(path string) (*sql.DB, error) {
+	for _, p := range []string{path, path + "-wal", path + "-shm"} {
+		if _, err := os.Stat(p); err == nil {
+			return nil, fmt.Errorf("%s already exists; refusing to overwrite", p)
+		}
+	}
+	if fi, err := os.Stat(filepath.Dir(path)); err != nil || !fi.IsDir() {
+		return nil, fmt.Errorf("output directory %s does not exist", filepath.Dir(path))
+	}
+	tmp, err := os.MkdirTemp("", "claims-migrate-cache-")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmp)
+	c := storage.NewClient()
+	if err := c.Connect(path, filepath.Join(tmp, "cache.db")); err != nil {
+		removeDB(path)
+		return nil, err
+	}
+	if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
+		c.Close()
+		removeDB(path)
+		return nil, err
+	}
+	if err := c.Close(); err != nil {
+		removeDB(path)
+		return nil, err
+	}
+	db, err := sql.Open("sqlite", "file:"+path+"?_pragma=journal_mode(WAL)&_pragma=synchronous(OFF)&_pragma=busy_timeout(30000)&_pragma=cache_size(-262144)")
+	if err != nil {
+		removeDB(path)
+		return nil, err
+	}
+	db.SetMaxOpenConns(1)
+	return db, nil
+}
+
+// checkTruth reopens the result the way meta-core boots it (schema gate).
+func checkTruth(path string) error {
+	tmp, err := os.MkdirTemp("", "claims-migrate-check-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmp)
+	c := storage.NewClient()
+	if err := c.Connect(path, filepath.Join(tmp, "cache.db")); err != nil {
+		return err
+	}
+	defer c.Close()
+	return c.EnsureSchemaVersion(storage.SchemaVersion)
+}
+
+func removeDB(path string) {
+	for _, p := range []string{path, path + "-wal", path + "-shm"} {
+		_ = os.Remove(p)
+	}
+}
+
+// printSummary writes the human report.
+func printSummary(w io.Writer, st *stats, dryRun bool) {
+	mode := "written"
+	if dryRun {
+		mode = "would be written (dry run: signed + verified, nothing stored)"
+	}
+	fmt.Fprintf(w, "\n=== claims-migrate summary ===\n")
+	fmt.Fprintf(w, "legacy signer uid        %s\n", st.LegacyUID)
+	fmt.Fprintf(w, "conversion ts (ms)       %d\n", st.ConvTS)
+	fmt.Fprintf(w, "records (rids) migrated  %d  (%d with no claims: only cids / watcher facts)\n", st.Rids, st.RidsNoClaims)
+	fmt.Fprintf(w, "records dropped (no cid) %d\n", st.RecordsNoCID)
+	fmt.Fprintf(w, "claims %-17s %d\n", "total", st.ClaimsWritten)
+	fmt.Fprintf(w, "  legacy field claims    %d\n", st.FieldClaims)
+	fmt.Fprintf(w, "  legacy alias claims    %d  (cids/<other> = \"true\")\n", st.AliasClaims)
+	fmt.Fprintf(w, "  UDL claims             %d  (%d signers)\n", st.UDLMigrated.total(), st.UDLSigners)
+	printCounts(w, "    ", st.UDLMigrated)
+	fmt.Fprintf(w, "  -> %s\n", mode)
+	fmt.Fprintf(w, "principals               %d\n", st.Principals)
+	fmt.Fprintf(w, "cid_group rows           %d  (%d minted for UDL-only cids)\n", st.CIDGroupRows, st.GroupsMinted)
+	fmt.Fprintf(w, "cid conflicts            %d  (cids/<c> on a rid while cid:<c> names another; index wins)\n", st.CIDConflicts)
+	fmt.Fprintf(w, "orphan cid: rows dropped %d  (rid has no file: rows)\n", st.OrphanCIDRows)
+	fmt.Fprintf(w, "watcher facts skipped    %d\n", st.WatcherFacts.total())
+	printCounts(w, "    ", st.WatcherFacts)
+	fmt.Fprintf(w, "fields refused           %d\n", st.RefusedFields.total())
+	printCounts(w, "    ", st.RefusedFields)
+	fmt.Fprintf(w, "malformed file: keys     %d\n", st.Malformed)
+	fmt.Fprintf(w, "UDL cells dropped        %d\n", st.UDLDropped.total())
+	printCounts(w, "    ", st.UDLDropped)
+	fmt.Fprintf(w, "scratch keys dropped     %d  (refetched / rebuilt in the cache)\n", st.Scratch.total())
+	printCounts(w, "    ", st.Scratch)
+	fmt.Fprintf(w, "derived keys skipped     %d\n", st.Derived.total())
+	printCounts(w, "    ", st.Derived)
+	fmt.Fprintf(w, "elapsed                  %s\n", st.Elapsed.Round(time.Millisecond))
+}
+
+func printCounts(w io.Writer, indent string, c counts) {
+	keys := make([]string, 0, len(c))
+	for k := range c {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		fmt.Fprintf(w, "%s%-40s %d\n", indent, k, c[k])
+	}
+}

tools/claims-migrate/migrate_test.go

@@ -0,0 +1,375 @@
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"encoding/base64"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"math"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	"github.com/metazla/meta-core/internal/claim"
+	"github.com/metazla/meta-core/internal/identity"
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+const (
+	cidIPFS  = "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 40
+	cidSHA   = "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" // rank 30
+	cidB     = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
+	cidOrph  = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
+	ridA     = "01JKR8XW5T4QMV7AHNJ2DEFGHK"
+	ridB     = "01JKR8XW5T4QMV7AHNJ2DEFGHM"
+	ridC     = "01JKR8XW5T4QMV7AHNJ2DEFGHN"
+	ridD     = "01JKR8XW5T4QMV7AHNJ2DEFGHP"
+	oldTSsec = 1_700_000_000
+)
+
+// --- minimal canonical CBOR encoder for building MMUR1 fixtures ---
+
+func head(major byte, n uint64) []byte {
+	m := major << 5
+	switch {
+	case n < 24:
+		return []byte{m | byte(n)}
+	case n <= 0xff:
+		return []byte{m | 24, byte(n)}
+	case n <= 0xffff:
+		return []byte{m | 25, byte(n >> 8), byte(n)}
+	case n <= 0xffffffff:
+		return binary.BigEndian.AppendUint32([]byte{m | 26}, uint32(n))
+	}
+	return binary.BigEndian.AppendUint64([]byte{m | 27}, n)
+}
+
+func encText(s string) []byte  { return append(head(3, uint64(len(s))), s...) }
+func encBytes(b []byte) []byte { return append(head(2, uint64(len(b))), b...) }
+func encUint(n uint64) []byte  { return head(0, n) }
+func encBool(b bool) []byte {
+	if b {
+		return []byte{0xf5}
+	}
+	return []byte{0xf4}
+}
+func encF64(f float64) []byte {
+	return binary.BigEndian.AppendUint64([]byte{0xfb}, math.Float64bits(f))
+}
+
+var encNull = []byte{0xf6}
+
+// mmur1 builds a signed MMUR1 record; badSig signs other bytes.
+func mmur1(t *testing.T, id *identity.Identity, cid, key string, value []byte, version uint64, badSig bool) []byte {
+	t.Helper()
+	var elems []byte
+	elems = append(elems, encText(cid)...)
+	elems = append(elems, encText(id.UID)...)
+	elems = append(elems, encText(key)...)
+	elems = append(elems, value...)
+	elems = append(elems, encUint(version)...)
+	elems = append(elems, encUint(oldTSsec)...)
+	signed := append(append([]byte(mmur1Domain), 0x86), elems...)
+	if badSig {
+		signed = append(signed, 'x')
+	}
+	sig, err := identity.Sign(id, signed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return append(append([]byte{0x87}, elems...), encBytes(sig)...)
+}
+
+func cell(t *testing.T, rec []byte, version uint64) string {
+	b, err := json.Marshal(map[string]any{
+		"record": base64.StdEncoding.EncodeToString(rec), "version": version, "ts": oldTSsec,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}
+
+func importID(t *testing.T, n byte) *identity.Identity {
+	t.Helper()
+	id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 7))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+type fixture struct {
+	dir, oldPath, accounts string
+	user, stranger         *identity.Identity
+	seekPlain, durPlain    []byte
+}
+
+func buildV1(t *testing.T) fixture {
+	t.Helper()
+	dir := t.TempDir()
+	f := fixture{dir: dir, oldPath: filepath.Join(dir, "v1.db"), accounts: filepath.Join(dir, "accounts")}
+	f.user, f.stranger = importID(t, 0x11), importID(t, 0x22)
+	if err := os.MkdirAll(f.accounts, 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := identity.SaveAccount(f.accounts, f.user); err != nil {
+		t.Fatal(err)
+	}
+	aead, err := identity.DeriveAEADKey(f.user)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f.seekPlain = encF64(123.5)
+	f.durPlain = encF64(5400)
+	env, err := sealEnvelope(aead, f.seekPlain, envelopeAAD(cidIPFS, f.user.UID, "seek", 3), bytes.Repeat([]byte{9}, 24))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db, err := sql.Open("sqlite", "file:"+f.oldPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
+		t.Fatal(err)
+	}
+	u := f.user.UID
+	rows := map[string]string{
+		"meta-core:schema-version": "2",
+		// record A: two cids, canonical = the dag-pb one
+		"file:" + ridA + "/cids/" + cidIPFS:   "true",
+		"file:" + ridA + "/cids/" + cidSHA:    "true",
+		"file:" + ridA + "/title":             "Inception",
+		"file:" + ridA + "/year":              "007",
+		"file:" + ridA + "/sizeByte":          "15032385536",
+		"file:" + ridA + "/genres/Drama":      "true",
+		"file:" + ridA + "/filePath":          "/files/watch/Inception.mkv",
+		"file:" + ridA + "/mtimeNano":         "1729872000000000000",
+		"file:" + ridA + "/cid_sha256":        cidSHA,
+		"file:" + ridA + "/fileinfo/duration": "8878.5",
+		"cid:" + cidIPFS:                      ridA,
+		"cid:" + cidSHA:                       ridA,
+		// record B: its cid only in the index; claims cidSHA too (conflict)
+		"file:" + ridB + "/title":          "Other",
+		"file:" + ridB + "/cids/" + cidSHA: "true",
+		"cid:" + cidB:                      ridB,
+		// record C: no cid at all → dropped
+		"file:" + ridC + "/title": "Nobody",
+		// orphan index row
+		"cid:" + cidOrph: ridD,
+		// scratch + derived
+		"btmagnet:abc":               "magnet:?xt=urn:btih:abc",
+		"searchcov:q":                "1",
+		"nzb:x":                      "y",
+		"idx:roots/" + ridA:          "",
+		"dup:" + ridA + "/p":         "",
+		"udl:idx:user:" + u + "\x1f": "",
+		// UDL
+		"udl:rec:" + u + "/" + cidIPFS + "/like":              cell(t, mmur1(t, f.user, cidIPFS, "like", encBool(true), 1, false), 1),
+		"udl:rec:" + u + "/" + cidIPFS + "/rating":            cell(t, mmur1(t, f.user, cidIPFS, "rating", encUint(4), 1, false), 1),
+		"udl:rec:" + u + "/" + cidIPFS + "/seek":              cell(t, mmur1(t, f.user, cidIPFS, "seek", encBytes(env), 3, false), 3),
+		"udl:rec:" + u + "/" + cidIPFS + "/watched":           cell(t, mmur1(t, f.user, cidIPFS, "watched", encNull, 2, false), 2),
+		"udl:rec:" + u + "/" + cidIPFS + "/duration":          cell(t, mmur1(t, f.user, cidIPFS, "duration", f.durPlain, 1, false), 1),
+		"udl:rec:" + u + "/self/profile:name":                 cell(t, mmur1(t, f.user, "self", "profile:name", encText("Alice"), 1, false), 1),
+		"udl:rec:" + u + "/" + cidSHA + "/like":               cell(t, mmur1(t, f.user, cidSHA, "like", encBool(true), 1, true), 1),
+		"udl:rec:" + f.stranger.UID + "/" + cidIPFS + "/like": cell(t, mmur1(t, f.stranger, cidIPFS, "like", encBool(false), 1, false), 1),
+	}
+	for k, v := range rows {
+		if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return f
+}
+
+func TestMigrate(t *testing.T) {
+	f := buildV1(t)
+	truthPath := filepath.Join(f.dir, "truth.db")
+	now := time.UnixMilli(1_790_000_000_000)
+	st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, now: now, workers: 3})
+	if err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	var sum bytes.Buffer
+	printSummary(&sum, st, false)
+	t.Log(sum.String())
+
+	check := func(name string, got, want int) {
+		t.Helper()
+		if got != want {
+			t.Errorf("%s = %d, want %d", name, got, want)
+		}
+	}
+	check("rids", st.Rids, 2)
+	check("records without cid", st.RecordsNoCID, 1)
+	check("orphan cid rows", st.OrphanCIDRows, 1)
+	check("cid conflicts", st.CIDConflicts, 1)
+	check("alias claims", st.AliasClaims, 1)
+	check("field claims", st.FieldClaims, 6) // A: title year sizeByte genres/Drama fileinfo/duration; B: title
+	check("refused cid_*", st.RefusedFields["cid_*"], 1)
+	check("watcher facts", st.WatcherFacts.total(), 2)
+	check("udl migrated", st.UDLMigrated.total(), 6)
+	check("udl no account", st.UDLDropped["no account on this box"], 1)
+	check("udl bad sig", st.UDLDropped["bad MMUR1 signature"], 1)
+	check("groups minted", st.GroupsMinted, 1) // "self"
+	check("cid_group rows", st.CIDGroupRows, 4)
+	check("scratch", st.Scratch.total(), 3)
+	check("principals", st.Principals, 2)
+	check("claims", st.ClaimsWritten, st.FieldClaims+st.AliasClaims+st.UDLMigrated.total())
+
+	// Reopen with meta-core's storage package: boot catch-up rebuilds the cache.
+	c := storage.NewClient()
+	if err := c.Connect(truthPath, filepath.Join(f.dir, "cache.db")); err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
+		t.Fatal(err)
+	}
+	if err := c.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	for cid, want := range map[string]string{cidIPFS: ridA, cidSHA: ridA, cidB: ridB, cidOrph: ""} {
+		got, err := c.GetByCID(cid)
+		if err != nil || got != want {
+			t.Errorf("GetByCID(%s) = %q, %v; want %q", cid, got, err, want)
+		}
+	}
+	a, err := c.GetMetadataFlat(ridA)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for k, want := range map[string]string{
+		"title": "Inception", "year": "007", "sizeByte": "15032385536", "genres/Drama": "true",
+		"fileinfo/duration": "8878.5", "cids/" + cidIPFS: "true", "cids/" + cidSHA: "true",
+	} {
+		if a[k] != want {
+			t.Errorf("record A %s = %q, want %q", k, a[k], want)
+		}
+	}
+	for _, k := range []string{"filePath", "mtimeNano", "cid_sha256"} {
+		if _, ok := a[k]; ok {
+			t.Errorf("record A carries %s", k)
+		}
+	}
+	b, err := c.GetMetadataFlat(ridB)
+	if err != nil || b["title"] != "Other" {
+		t.Errorf("record B = %v, %v", b, err)
+	}
+	ids, err := c.GetAllHashIDs()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Contains(ids, ridA) || !slices.Contains(ids, ridB) || slices.Contains(ids, ridC) || slices.Contains(ids, ridD) {
+		t.Errorf("roots = %v", ids)
+	}
+	if len(ids) != 3 {
+		t.Errorf("want 3 roots (A, B, self), got %v", ids)
+	}
+
+	// Legacy claims sit on the canonical cid, signed by the printed uid.
+	cls, err := c.ClaimsForCIDs([]string{cidIPFS}, "title")
+	if err != nil || len(cls) != 1 || cls[0].UID != st.LegacyUID || cls[0].TS != uint64(now.UnixMilli()) {
+		t.Fatalf("title claims = %+v, %v", cls, err)
+	}
+	if err := cls[0].Verify(); err != nil {
+		t.Error(err)
+	}
+	alias, err := c.ClaimsForCIDs([]string{cidIPFS}, "cids/"+cidSHA)
+	if err != nil || len(alias) != 1 || alias[0].Value.Text != "true" {
+		t.Errorf("alias claims = %+v, %v", alias, err)
+	}
+
+	// UDL claims.
+	u := f.user.UID
+	wantTS := uint64(oldTSsec) * 1000
+	get := func(cid, key string) claim.Claim {
+		t.Helper()
+		cl, ok, err := c.UDLGetClaim(u, cid, key)
+		if err != nil || !ok {
+			t.Fatalf("UDL %s/%s: ok=%v err=%v", cid, key, ok, err)
+		}
+		if cl.TS != wantTS {
+			t.Errorf("UDL %s ts = %d, want %d", key, cl.TS, wantTS)
+		}
+		if err := cl.Verify(); err != nil {
+			t.Errorf("UDL %s: %v", key, err)
+		}
+		return cl
+	}
+	if v := get(cidIPFS, "like").Value; v.Kind != claim.KindText || v.Text != "true" {
+		t.Errorf("like = %+v", v)
+	}
+	if v := get(cidIPFS, "userRating").Value; v.Kind != claim.KindText || v.Text != "4" {
+		t.Errorf("userRating = %+v", v)
+	}
+	if _, ok, _ := c.UDLGetClaim(u, cidIPFS, "rating"); ok {
+		t.Error("rating was not renamed")
+	}
+	if v := get(cidIPFS, "watched").Value; !v.IsNull() {
+		t.Errorf("watched tombstone = %+v", v)
+	}
+	if v := get("self", "profile:name").Value; v.Text != "Alice" {
+		t.Errorf("profile:name = %+v", v)
+	}
+	aead, _ := identity.DeriveAEADKey(f.user)
+	for key, plain := range map[string][]byte{"seek": f.seekPlain, "duration": f.durPlain} {
+		cl := get(cidIPFS, key)
+		if cl.Value.Kind != claim.KindBytes {
+			t.Fatalf("%s is not an envelope: %+v", key, cl.Value)
+		}
+		pt, err := openEnvelope(aead, cl.Value.Bytes, envelopeAAD(cidIPFS, u, key, cl.TS))
+		if err != nil || !bytes.Equal(pt, plain) {
+			t.Errorf("%s decrypts to %x, %v; want %x", key, pt, err, plain)
+		}
+	}
+	if _, ok, _ := c.UDLGetClaim(f.stranger.UID, cidIPFS, "like"); ok {
+		t.Error("stranger's cell migrated without an account")
+	}
+
+	// A second run refuses to overwrite.
+	if _, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts}); err == nil {
+		t.Error("second run overwrote the truth file")
+	}
+}
+
+func TestDryRunWritesNothing(t *testing.T) {
+	f := buildV1(t)
+	truthPath := filepath.Join(f.dir, "truth.db")
+	st, err := migrate(options{oldPath: f.oldPath, truthPath: truthPath, accountsDir: f.accounts, dryRun: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := os.Stat(truthPath); !os.IsNotExist(err) {
+		t.Errorf("dry run created %s", truthPath)
+	}
+	if st.ClaimsWritten != st.FieldClaims+st.AliasClaims+st.UDLMigrated.total() || st.ClaimsWritten == 0 {
+		t.Errorf("dry-run counts: %+v", st)
+	}
+}
+
+func TestNoAccountsDropsUDL(t *testing.T) {
+	f := buildV1(t)
+	st, err := migrate(options{oldPath: f.oldPath, dryRun: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.UDLMigrated.total() != 0 || st.UDLDropped["no account on this box"] != 7 {
+		t.Errorf("udl: migrated %v dropped %v", st.UDLMigrated, st.UDLDropped)
+	}
+}
+
+func TestHalfFloat(t *testing.T) {
+	for h, want := range map[uint16]float64{0x3c00: 1, 0xc000: -2, 0x3e00: 1.5, 0x0000: 0, 0x7bff: 65504} {
+		if got := halfToFloat(h); got != want {
+			t.Errorf("half %04x = %v, want %v", h, got, want)
+		}
+	}
+}

tools/claims-migrate/mmur1.go

@@ -0,0 +1,267 @@
+package main
+
+// Decoding of the retired MMUR1 user-data record (meta-watch record/canonical.rs
+// at meta-core v1.2.0) and the private-tier envelope (record/crypto.rs).
+//
+//	record        = canonical CBOR array(7) [cid, uid, key, value, version, ts, sig]
+//	bytes_to_sign = "MMUR1\n" || array(6) [cid, uid, key, value, version, ts]
+//	sig           = DER ECDSA-secp256k1 over SHA-256(bytes_to_sign)
+//
+// value is CBOR null / bool / int / float / text / bytes (bytes = a private-tier
+// XChaCha20-Poly1305 envelope whose plaintext is the CBOR of the typed value).
+// The signed bytes are rebuilt from the record's own raw element bytes (array
+// header 0x87 → 0x86, sig element cut off), so the old signature is checked
+// against exactly what was signed, whatever float width the encoder chose.
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+const mmur1Domain = "MMUR1\n"
+
+type cborKind uint8
+
+const (
+	cNull cborKind = iota
+	cBool
+	cInt
+	cFloat
+	cText
+	cBytes
+)
+
+// cborItem is one decoded scalar plus its raw encoding.
+type cborItem struct {
+	kind  cborKind
+	b     bool
+	i     int64
+	f     float64
+	s     string
+	bytes []byte
+	raw   []byte
+}
+
+// wireText is the MMUR2 text form of a typed value — meta-watch's
+// Value::to_wire_text: bools "true"/"false", ints in decimal, floats as Rust's
+// f64 Display (shortest round-trip, never an exponent), text verbatim.
+func (it cborItem) wireText() (string, bool) {
+	switch it.kind {
+	case cBool:
+		if it.b {
+			return "true", true
+		}
+		return "false", true
+	case cInt:
+		return strconv.FormatInt(it.i, 10), true
+	case cFloat:
+		return strconv.FormatFloat(it.f, 'f', -1, 64), true
+	case cText:
+		return it.s, true
+	}
+	return "", false
+}
+
+type mmur1Record struct {
+	CID, UID, Key string
+	Value         cborItem
+	Version       uint64
+	TS            int64
+	Sig           []byte
+	Signed        []byte // the exact MMUR1 bytes the signature covers
+}
+
+var errCBOR = errors.New("malformed MMUR1 CBOR")
+
+// readHead parses a CBOR initial byte + argument (definite lengths only).
+func readHead(b []byte, off int) (major byte, ai byte, n uint64, next int, err error) {
+	if off >= len(b) {
+		return 0, 0, 0, 0, errCBOR
+	}
+	ib := b[off]
+	major, ai = ib>>5, ib&31
+	off++
+	var size int
+	switch {
+	case ai < 24:
+		return major, ai, uint64(ai), off, nil
+	case ai == 24:
+		size = 1
+	case ai == 25:
+		size = 2
+	case ai == 26:
+		size = 4
+	case ai == 27:
+		size = 8
+	default:
+		return 0, 0, 0, 0, fmt.Errorf("%w: indefinite or reserved length", errCBOR)
+	}
+	if off+size > len(b) {
+		return 0, 0, 0, 0, errCBOR
+	}
+	for _, x := range b[off : off+size] {
+		n = n<<8 | uint64(x)
+	}
+	return major, ai, n, off + size, nil
+}
+
+// readItem decodes one scalar item at off.
+func readItem(b []byte, off int) (cborItem, int, error) {
+	start := off
+	major, ai, n, off, err := readHead(b, off)
+	if err != nil {
+		return cborItem{}, 0, err
+	}
+	var it cborItem
+	switch major {
+	case 0:
+		if n > math.MaxInt64 {
+			return it, 0, fmt.Errorf("%w: int out of range", errCBOR)
+		}
+		it = cborItem{kind: cInt, i: int64(n)}
+	case 1:
+		if n > math.MaxInt64 {
+			return it, 0, fmt.Errorf("%w: int out of range", errCBOR)
+		}
+		it = cborItem{kind: cInt, i: -1 - int64(n)}
+	case 2, 3:
+		if n > uint64(len(b)-off) {
+			return it, 0, errCBOR
+		}
+		end := off + int(n)
+		if major == 2 {
+			it = cborItem{kind: cBytes, bytes: append([]byte(nil), b[off:end]...)}
+		} else {
+			it = cborItem{kind: cText, s: string(b[off:end])}
+		}
+		off = end
+	case 7:
+		switch ai {
+		case 20, 21:
+			it = cborItem{kind: cBool, b: ai == 21}
+		case 22:
+			it = cborItem{kind: cNull}
+		case 25:
+			it = cborItem{kind: cFloat, f: halfToFloat(uint16(n))}
+		case 26:
+			it = cborItem{kind: cFloat, f: float64(math.Float32frombits(uint32(n)))}
+		case 27:
+			it = cborItem{kind: cFloat, f: math.Float64frombits(n)}
+		default:
+			return it, 0, fmt.Errorf("%w: unsupported simple value %d", errCBOR, ai)
+		}
+	default:
+		return it, 0, fmt.Errorf("%w: unsupported major type %d", errCBOR, major)
+	}
+	it.raw = b[start:off]
+	return it, off, nil
+}
+
+func halfToFloat(h uint16) float64 {
+	sign := 1.0
+	if h&0x8000 != 0 {
+		sign = -1
+	}
+	exp := int(h>>10) & 0x1f
+	frac := float64(h & 0x3ff)
+	switch exp {
+	case 0:
+		return sign * math.Ldexp(frac, -24)
+	case 31:
+		if frac == 0 {
+			return math.Inf(int(sign))
+		}
+		return math.NaN()
+	}
+	return sign * math.Ldexp(frac+1024, exp-25)
+}
+
+// decodeMMUR1 parses a stored record and rebuilds its signed bytes.
+func decodeMMUR1(b []byte) (*mmur1Record, error) {
+	if len(b) == 0 || b[0] != 0x87 {
+		return nil, fmt.Errorf("%w: want array(7)", errCBOR)
+	}
+	off := 1
+	items := make([]cborItem, 7)
+	sigStart := 0
+	for i := range items {
+		if i == 6 {
+			sigStart = off
+		}
+		it, next, err := readItem(b, off)
+		if err != nil {
+			return nil, err
+		}
+		items[i], off = it, next
+	}
+	if off != len(b) {
+		return nil, fmt.Errorf("%w: trailing bytes", errCBOR)
+	}
+	for i, want := range []cborKind{cText, cText, cText} {
+		if items[i].kind != want {
+			return nil, fmt.Errorf("%w: cid/uid/key must be text", errCBOR)
+		}
+	}
+	if items[4].kind != cInt || items[4].i < 0 {
+		return nil, fmt.Errorf("%w: version must be an unsigned int", errCBOR)
+	}
+	if items[5].kind != cInt {
+		return nil, fmt.Errorf("%w: ts must be an int", errCBOR)
+	}
+	if items[6].kind != cBytes {
+		return nil, fmt.Errorf("%w: sig must be bytes", errCBOR)
+	}
+	signed := make([]byte, 0, len(mmur1Domain)+sigStart)
+	signed = append(signed, mmur1Domain...)
+	signed = append(signed, 0x86)
+	signed = append(signed, b[1:sigStart]...)
+	return &mmur1Record{
+		CID: items[0].s, UID: items[1].s, Key: items[2].s,
+		Value:   items[3],
+		Version: uint64(items[4].i),
+		TS:      items[5].i,
+		Sig:     items[6].bytes,
+		Signed:  signed,
+	}, nil
+}
+
+// envelopeAAD is cid ‖ 0x1F ‖ uid ‖ 0x1F ‖ key ‖ 0x1F ‖ n_be(8) — n was the
+// MMUR1 version and is the MMUR2 ts.
+func envelopeAAD(cid, uid, key string, n uint64) []byte {
+	b := make([]byte, 0, len(cid)+len(uid)+len(key)+11)
+	b = append(b, cid...)
+	b = append(b, 0x1f)
+	b = append(b, uid...)
+	b = append(b, 0x1f)
+	b = append(b, key...)
+	b = append(b, 0x1f)
+	return binary.BigEndian.AppendUint64(b, n)
+}
+
+// openEnvelope decrypts nonce(24) ‖ ciphertext+tag.
+func openEnvelope(aeadKey, env, aad []byte) ([]byte, error) {
+	a, err := chacha20poly1305.NewX(aeadKey)
+	if err != nil {
+		return nil, err
+	}
+	if len(env) < chacha20poly1305.NonceSizeX+a.Overhead() {
+		return nil, errors.New("envelope too short")
+	}
+	return a.Open(nil, env[:chacha20poly1305.NonceSizeX], env[chacha20poly1305.NonceSizeX:], aad)
+}
+
+// sealEnvelope encrypts plain under a fresh random nonce.
+func sealEnvelope(aeadKey, plain, aad []byte, nonce []byte) ([]byte, error) {
+	a, err := chacha20poly1305.NewX(aeadKey)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]byte, 0, len(nonce)+len(plain)+a.Overhead())
+	out = append(out, nonce...)
+	return a.Seal(out, nonce, plain, aad), nil
+}

tools/search-parity/README.md

@@ -1,21 +1,23 @@
 # search-parity
 
 Regression check for `POST /api/metadata/search` against a running meta-core.
+Since v2.0.0 it reads the merged view from the cache database
+(`/meta-core/db-cache/cache.db`), which keeps the v1 `file:<rid>/<field>` layout.
 
-- `parity.py <meta-core.db>` builds ~230 queries from real records (free text,
+- `parity.py <cache.db>` builds ~230 queries from real records (free text,
   short tokens, key-set and flat filters, exact/contains, multi-filter). It
   computes each query's expected result by brute force: one streaming pass
   over every record, using a port of the Go matchers. Then it diffs that
   against the live API. Any `lost` result means the SQL candidate superset
   dropped a real match.
-- `timing.py [meta-core.db]` times the same query set at limit 100 and 2000.
+- `timing.py [cache.db]` times the same query set at limit 100 and 2000.
 
 Run them inside the container. It has python3 and reads the database
 read-only:
 
 ```bash
 docker cp tools/search-parity metacore-app:/tmp/sp
-docker exec metacore-app python3 -u /tmp/sp/parity.py /meta-core/db/meta-core.db
+docker exec metacore-app python3 -u /tmp/sp/parity.py /meta-core/db-cache/cache.db
 docker exec metacore-app python3 -u /tmp/sp/timing.py
 ```
 

tools/search-parity/parity.py

@@ -1,6 +1,10 @@
-"""Search parity check for the meta-core Redis -> SQLite cut-over.
+"""Search parity check for meta-core search.
 
-  parity.py <meta-core.db>
+  parity.py <cache.db>
+
+Since v2.0.0 the searchable records are the MERGED view, kept in the cache
+database's kv table under the v1 layout (file:<rid>/<field>, idx:roots/<rid>):
+point this at /meta-core/db-cache/cache.db, not the truth database.
 
 Builds a query set from real records, computes the EXPECTED result of each
 query by brute force — one streaming pass over every record in the database,

tools/search-parity/timing.py

@@ -1,6 +1,6 @@
 import importlib.util, json, sqlite3, sys, time
 spec = importlib.util.spec_from_file_location("parity", "/tmp/parity.py")
-sys.argv = ["parity", "/meta-core/db/meta-core.db"]
+sys.argv = ["parity", sys.argv[1] if len(sys.argv) > 1 else "/meta-core/db-cache/cache.db"]
 src = open("/tmp/parity.py").read().replace("\nmain()\n", "\n")
 ns = {}
 exec(compile(src, "parity", "exec"), ns)