migrate

tools/claims-migrate/migrate.go

@@ -61,6 +61,9 @@ type stats struct {
 
 	CIDGroupRows  int
 	CIDConflicts  int // cids/<c> on a rid while cid:<c> names another rid
+	CIDShared     int // cids/<c> (no cid: row) already kept by an earlier record
+	RecordsFolded int // records whose every cid an earlier record kept: merged into it
+	DupClaims     int // (cid, key) already claimed by the legacy signer (folded records)
 	OrphanCIDRows int // cid:<c> → rid with no file: rows
 
 	UDLMigrated  counts
@@ -74,6 +77,8 @@ type stats struct {
 	Principals    int
 	ClaimsWritten int
 	Elapsed       time.Duration
+
+	canon map[string]string // rid → the canonical cid its claims sit on
 }
 
 func newStats() *stats {
@@ -81,6 +86,7 @@ func newStats() *stats {
 		WatcherFacts: counts{}, RefusedFields: counts{},
 		UDLMigrated: counts{}, UDLDropped: counts{},
 		Scratch: counts{}, Derived: counts{},
+		canon: map[string]string{},
 	}
 }
 
@@ -239,6 +245,7 @@ func migrate(o options) (st *stats, err error) {
 	st.CIDGroupRows = len(groups)
 	st.Principals = len(w.pids)
 	st.ClaimsWritten = int(w.seq)
+	st.DupClaims = w.dups
 	if !o.dryRun {
 		if err = w.finish(groups); err != nil {
 			return nil, err
@@ -439,14 +446,40 @@ func emitRecord(o options, st *stats, legacy *identity.Identity, index map[strin
 		st.RecordsNoCID++
 		return nil
 	}
-	canonical := usable[0]
-	for _, c := range usable[1:] {
-		canonical = cid.Better(canonical, c)
-	}
+	// A cid with no cid: index row can sit in two records' cids/* (the same
+	// content stored twice). The first record keeps it; a record left with none
+	// folds into the record owning its first shared cid, and its fields become
+	// claims there (the legacy signer's first claim per (cid, key) wins).
+	owner, shared := "", ""
+	var own []string
 	for _, c := range usable {
-		groups[c] = rid
+		if r, ok := groups[c]; ok && r != rid {
+			st.CIDShared++
+			if owner == "" {
+				owner, shared = r, c
+			}
+			continue
+		}
+		own = append(own, c)
+	}
+	var canonical string
+	if len(own) == 0 {
+		st.RecordsFolded++
+		rid = owner
+		if canonical = st.canon[owner]; canonical == "" {
+			canonical = shared
+		}
+	} else {
+		canonical = own[0]
+		for _, c := range own[1:] {
+			canonical = cid.Better(canonical, c)
+		}
+		for _, c := range own {
+			groups[c] = rid
+		}
+		st.canon[rid] = canonical
+		st.Rids++
 	}
-	st.Rids++
 	emitted := 0
 	mk := func(key, value string) error {
 		emitted++
@@ -685,6 +718,7 @@ type writer struct {
 	db   *sql.DB
 	pids map[string]int64
 	seq  int64
+	dups int
 }
 
 func (w *writer) write(batch []claim.Claim) error {
@@ -702,7 +736,7 @@ func (w *writer) write(batch []claim.Claim) error {
 		return err
 	}
 	defer tx.Rollback()
-	ins, err := tx.Prepare(`INSERT INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`)
+	ins, err := tx.Prepare(`INSERT OR IGNORE INTO claim(cid, key, pid, value, ts, sig, seq) VALUES(?, ?, ?, ?, ?, ?, ?)`)
 	if err != nil {
 		return err
 	}
@@ -716,9 +750,14 @@ func (w *writer) write(batch []claim.Claim) error {
 			w.pids[cl.UID] = pid
 		}
 		w.seq++
-		if _, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq); err != nil {
+		res, err := ins.Exec(cl.CID, cl.Key, pid, valueArg(cl.Value), int64(cl.TS), cl.Sig, w.seq)
+		if err != nil {
 			return fmt.Errorf("claim (%s, %s, %s): %w", cl.CID, cl.Key, cl.UID, err)
 		}
+		if n, _ := res.RowsAffected(); n == 0 { // a folded record's duplicate
+			w.seq--
+			w.dups++
+		}
 	}
 	return tx.Commit()
 }
@@ -875,6 +914,8 @@ func printSummary(w io.Writer, st *stats, dryRun bool) {
 	fmt.Fprintf(w, "principals               %d\n", st.Principals)
 	fmt.Fprintf(w, "cid_group rows           %d  (%d minted for UDL-only cids)\n", st.CIDGroupRows, st.GroupsMinted)
 	fmt.Fprintf(w, "cid conflicts            %d  (cids/<c> on a rid while cid:<c> names another; index wins)\n", st.CIDConflicts)
+	fmt.Fprintf(w, "shared cids              %d  (cids/<c> kept by an earlier record; %d records folded into it)\n", st.CIDShared, st.RecordsFolded)
+	fmt.Fprintf(w, "duplicate claims skipped %d  (folded records' (cid, key) already claimed; not counted in dry run)\n", st.DupClaims)
 	fmt.Fprintf(w, "orphan cid: rows dropped %d  (rid has no file: rows)\n", st.OrphanCIDRows)
 	fmt.Fprintf(w, "watcher facts skipped    %d\n", st.WatcherFacts.total())
 	printCounts(w, "    ", st.WatcherFacts)