diff --git a/tools/claims-migrate/migrate.go b/tools/claims-migrate/migrate.go index 047ddd6..f050b1f 100644 --- a/tools/claims-migrate/migrate.go +++ b/tools/claims-migrate/migrate.go @@ -61,6 +61,9 @@ type stats struct { CIDGroupRows int CIDConflicts int // cids/ on a rid while cid: names another rid + CIDShared int // cids/ (no cid: row) already kept by an earlier record + RecordsFolded int // records whose every cid an earlier record kept: merged into it + DupClaims int // (cid, key) already claimed by the legacy signer (folded records) OrphanCIDRows int // cid: → rid with no file: rows UDLMigrated counts @@ -74,6 +77,8 @@ type stats struct { Principals int ClaimsWritten int Elapsed time.Duration + + canon map[string]string // rid → the canonical cid its claims sit on } func newStats() *stats { @@ -81,6 +86,7 @@ func newStats() *stats { WatcherFacts: counts{}, RefusedFields: counts{}, UDLMigrated: counts{}, UDLDropped: counts{}, Scratch: counts{}, Derived: counts{}, + canon: map[string]string{}, } } @@ -239,6 +245,7 @@ func migrate(o options) (st *stats, err error) { st.CIDGroupRows = len(groups) st.Principals = len(w.pids) st.ClaimsWritten = int(w.seq) + st.DupClaims = w.dups if !o.dryRun { if err = w.finish(groups); err != nil { return nil, err @@ -439,14 +446,40 @@ func emitRecord(o options, st *stats, legacy *identity.Identity, index map[strin st.RecordsNoCID++ return nil } - canonical := usable[0] - for _, c := range usable[1:] { - canonical = cid.Better(canonical, c) - } + // A cid with no cid: index row can sit in two records' cids/* (the same + // content stored twice). The first record keeps it; a record left with none + // folds into the record owning its first shared cid, and its fields become + // claims there (the legacy signer's first claim per (cid, key) wins). + owner, shared := "", "" + var own []string for _, c := range usable { - groups[c] = rid + if r, ok := groups[c]; ok && r != rid { + st.CIDShared++ + if owner == "" { + owner, shared = r, c + } + continue + } + own = append(own, c) + } + var canonical string + if len(own) == 0 { + st.RecordsFolded++ + rid = owner + if canonical = st.canon[owner]; canonical == "" { + canonical = shared + } + } else { + canonical = own[0] + for _, c := range own[1:] { + canonical = cid.Better(canonical, c) + } + for _, c := range own { + groups[c] = rid + } + st.canon[rid] = canonical + st.Rids++ } - st.Rids++ emitted := 0 mk := func(key, value string) error { emitted++ @@ -685,6 +718,7 @@ type writer struct { db *sql.DB pids map[string]int64 seq int64 + dups int } func (w *writer) write(batch []claim.Claim) error { @@ -702,7 +736,7 @@ func (w *writer) write(batch []claim.Claim) error { return err } defer tx.Rollback() - ins, err := tx.Prepare(`INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`) + ins, err := tx.Prepare(`INSERT OR IGNORE INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`) if err != nil { return err } @@ -716,9 +750,14 @@ func (w *writer) write(batch []claim.Claim) error { w.pids[cl.UID] = pid } w.seq++ - if _, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq); err != nil { + res, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq) + if err != nil { return fmt.Errorf("claim (%s, %s, %s): %w", cl.CID, cl.Key, cl.UID, err) } + if n, _ := res.RowsAffected(); n == 0 { // a folded record's duplicate + w.seq-- + w.dups++ + } } return tx.Commit() } @@ -875,6 +914,8 @@ func printSummary(w io.Writer, st *stats, dryRun bool) { fmt.Fprintf(w, "principals %d\n", st.Principals) fmt.Fprintf(w, "cid_group rows %d (%d minted for UDL-only cids)\n", st.CIDGroupRows, st.GroupsMinted) fmt.Fprintf(w, "cid conflicts %d (cids/ on a rid while cid: names another; index wins)\n", st.CIDConflicts) + fmt.Fprintf(w, "shared cids %d (cids/ kept by an earlier record; %d records folded into it)\n", st.CIDShared, st.RecordsFolded) + fmt.Fprintf(w, "duplicate claims skipped %d (folded records' (cid, key) already claimed; not counted in dry run)\n", st.DupClaims) fmt.Fprintf(w, "orphan cid: rows dropped %d (rid has no file: rows)\n", st.OrphanCIDRows) fmt.Fprintf(w, "watcher facts skipped %d\n", st.WatcherFacts.total()) printCounts(w, " ", st.WatcherFacts) diff --git a/tools/claims-migrate/migrate_test.go b/tools/claims-migrate/migrate_test.go index 62d503c..bda4aa1 100644 --- a/tools/claims-migrate/migrate_test.go +++ b/tools/claims-migrate/migrate_test.go @@ -373,3 +373,71 @@ func TestHalfFloat(t *testing.T) { } } } + +// Two records listing the same cid in cids/* with no cid: index row (the same +// content stored twice): the first keeps it, a record left with no cid of its +// own folds into it, and the duplicate (cid, key) claims are skipped. +func TestSharedUnindexedCIDFolds(t *testing.T) { + dir := t.TempDir() + oldPath := filepath.Join(dir, "v1.db") + db, err := sql.Open("sqlite", "file:"+oldPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec(`CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { + t.Fatal(err) + } + for k, v := range map[string]string{ + "meta-core:schema-version": "2", + "file:" + ridA + "/cids/" + cidSHA: "true", + "file:" + ridA + "/title": "A", + "file:" + ridB + "/cids/" + cidSHA: "true", + "file:" + ridB + "/title": "B", + "file:" + ridB + "/genres/Drama": "true", + "file:" + ridC + "/cids/" + cidSHA: "true", + "file:" + ridC + "/cids/" + cidB: "true", + "file:" + ridC + "/title": "C", + } { + if _, err := db.Exec(`INSERT INTO kv(k, v) VALUES(?, ?)`, k, v); err != nil { + t.Fatal(err) + } + } + db.Close() + truthPath := filepath.Join(dir, "truth.db") + st, err := migrate(options{oldPath: oldPath, truthPath: truthPath, workers: 2}) + if err != nil { + t.Fatalf("migrate: %v", err) + } + if st.Rids != 2 || st.RecordsFolded != 1 || st.CIDShared != 2 || st.DupClaims != 1 { + t.Errorf("rids %d folded %d shared %d dups %d", st.Rids, st.RecordsFolded, st.CIDShared, st.DupClaims) + } + out, err := sql.Open("sqlite", "file:"+truthPath+"?mode=ro") + if err != nil { + t.Fatal(err) + } + defer out.Close() + groups := map[string]string{} + rows, err := out.Query(`SELECT cid, rid FROM cid_group`) + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var c, r string + if err := rows.Scan(&c, &r); err != nil { + t.Fatal(err) + } + groups[c] = r + } + rows.Close() + if groups[cidSHA] != ridA || groups[cidB] != ridC || len(groups) != 2 { + t.Errorf("cid_group %v", groups) + } + var n int + if err := out.QueryRow(`SELECT count(*) FROM claim WHERE cid = ? AND key IN ('title', 'genres/Drama')`, cidSHA).Scan(&n); err != nil || n != 2 { + t.Errorf("claims on the shared cid: %d (%v)", n, err) + } + var title string + if err := out.QueryRow(`SELECT value FROM claim WHERE cid = ? AND key = 'title'`, cidB).Scan(&title); err != nil || title != "C" { + t.Errorf("record C title on its own cid: %q (%v)", title, err) + } +}