v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/storage/client.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/metazla/meta-core/internal/kv"
+	"github.com/metazla/meta-core/internal/schema"
 )
 
 // Key layout (everything is one string→string table, see internal/kv):
@@ -21,6 +22,7 @@ import (
 //	cid:<cid>                reverse index → <rid>
 //	idx:roots/<rid>          the set of every root (was the Redis SET file:__index__)
 //	idx:fts/<rid>            rowid of the record's row in the FTS table
+//	idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
 //	dup:<rid>/<path>         a duplicate path of the record (was the SET file:<rid>/duplicates)
 //	udl:…                    User Data Layer, see udl.go
 //	ev:<stream>/…, evhwm:…   event streams, see internal/kv/stream.go
@@ -80,6 +82,7 @@ func (c *Client) Connect(path string) error {
 		ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
 		Readers:       envInt("META_CORE_DB_READERS", 8),
 		Schema:        searchSchema,
+		OnChange:      onChange,
 	})
 	if err != nil {
 		return fmt.Errorf("open %s: %w", path, err)
@@ -185,6 +188,101 @@ func isInternalKey(key string) bool {
 		strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS)
 }
 
+// Derived counters, maintained by onChange in the same transaction as the
+// change they count (kv.Tx.AddCounter): the root count behind search's `total`
+// and /api/stats, and the per-field schema behind /api/schema.
+const (
+	rootsCountKey  = "idx:count/roots"
+	countersReady  = "idx:count/ready"
+	counterRebuild = 5000
+)
+
+// onChange keeps the derived counters exact. It sees only real changes (kv
+// skips unchanged Puts) with the old and new values, so a modified field moves
+// one count from its old label to its new one.
+func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) {
+	if strings.HasPrefix(k, rootsNS) {
+		switch {
+		case hasNew && !hadOld:
+			tx.AddCounter(rootsCountKey, 1)
+		case hadOld && !hasNew:
+			tx.AddCounter(rootsCountKey, -1)
+		}
+		return
+	}
+	if !strings.HasPrefix(k, recordNS) {
+		return
+	}
+	if hadOld {
+		if ck, ok := schema.CounterKey(k, old); ok {
+			tx.AddCounter(ck, -1)
+		}
+	}
+	if hasNew {
+		if ck, ok := schema.CounterKey(k, v); ok {
+			tx.AddCounter(ck, 1)
+		}
+	}
+}
+
+// EnsureCounters builds the derived counters when they have never been built —
+// a store created before they existed, or one restored from outside meta-core.
+// Blocking and single-transaction (writes wait for it; ~1–2 min at millions of
+// keys, once), because a build racing live deltas would double-count.
+func (c *Client) EnsureCounters() (bool, error) {
+	s, err := c.kv()
+	if err != nil {
+		return false, err
+	}
+	ctx, cancel := opCtx()
+	_, ready, err := s.Get(ctx, countersReady)
+	cancel()
+	if err != nil || ready {
+		return false, err
+	}
+	return true, c.RebuildCounters()
+}
+
+// RebuildCounters recomputes every derived counter from the records, in one
+// transaction (writers wait for it).
+func (c *Client) RebuildCounters() error {
+	s, err := c.kv()
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	return s.Update(ctx, func(tx *kv.Tx) error {
+		for _, p := range []string{schema.CounterPrefix, rootsCountKey} {
+			if _, err := tx.DeletePrefix(p, 0); err != nil {
+				return err
+			}
+		}
+		roots, err := tx.Count(tx.Context(), rootsNS)
+		if err != nil {
+			return err
+		}
+		tx.AddCounter(rootsCountKey, roots)
+		after := ""
+		for {
+			rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild)
+			if err != nil {
+				return err
+			}
+			for _, row := range rows {
+				if ck, ok := schema.CounterKey(row.K, row.V); ok {
+					tx.AddCounter(ck, 1)
+				}
+			}
+			if len(rows) < counterRebuild {
+				break
+			}
+			after = rows[len(rows)-1].K
+		}
+		return tx.Put(countersReady, "1")
+	})
+}
+
 // readRecord returns a record's flat fields (nil when it has none).
 func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) {
 	prefix := recordPrefix(hashID)
@@ -210,17 +308,19 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
 		return fmt.Errorf("empty hash id")
 	}
 	prefix := recordPrefix(hashID)
-	touchesSearch := false
+	searchChanged := false
 	for field, value := range metadata {
 		if isReservedFieldName(field) {
 			continue
 		}
-		if err := tx.Put(prefix+field, value); err != nil {
+		changed, err := tx.Set(prefix+field, value)
+		if err != nil {
 			return err
 		}
-		touchesSearch = touchesSearch || affectsHaystack(field)
+		searchChanged = searchChanged || (changed && affectsHaystack(field))
 	}
-	if err := tx.Put(rootKey(hashID), ""); err != nil {
+	newRoot, err := tx.Set(rootKey(hashID), "")
+	if err != nil {
 		return err
 	}
 	for field := range metadata {
@@ -230,10 +330,12 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
 			}
 		}
 	}
-	if touchesSearch {
+	// The FTS row is rewritten only when the haystack can have changed — a
+	// re-persisted record whose values are all in place costs reads only.
+	if searchChanged || newRoot {
 		return refreshSearchTx(tx, hashID)
 	}
-	return ensureSearchRowTx(tx, hashID)
+	return nil
 }
 
 // GetMetadataFlat retrieves all metadata for a file as a flat map.

internal/storage/counters_test.go

@@ -0,0 +1,92 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/schema"
+)
+
+func schemaJSON(t *testing.T, c *Client) string {
+	t.Helper()
+	snap, err := schema.NewIndexer(c.Store()).Snapshot(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	snap.GeneratedAt = snap.GeneratedAt.Truncate(0).UTC()
+	b, _ := json.Marshal(snap.Fields)
+	return string(b)
+}
+
+// The counters behind /api/schema and search's `total` are maintained by the
+// write path. Whatever sequence of writes, modifications and deletions runs,
+// they must equal what a full rebuild from the records computes.
+func TestCountersStayExactAcrossWrites(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"R1": {"title": "One", "sizeByte": "100", "title/eng": "One"},
+		"R2": {"title": "Two", "sizeByte": "x", "genres/Action": "true"},
+		"R3": {"title": "Three", "sizeByte": "300"},
+	})
+	if err := c.SetProperty("R2", "sizeByte", "200"); err != nil { // label string → int
+		t.Fatal(err)
+	}
+	if err := c.SetProperty("R1", "title", "One"); err != nil { // unchanged: no delta
+		t.Fatal(err)
+	}
+	if err := c.DeleteProperty("R1", "title/eng"); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := c.DeleteMetadata("R3"); err != nil {
+		t.Fatal(err)
+	}
+
+	if n, err := c.CountRoots(); err != nil || n != 2 {
+		t.Fatalf("CountRoots = %d (err %v), want 2", n, err)
+	}
+	incremental := schemaJSON(t, c)
+	snap, _ := schema.NewIndexer(c.Store()).Snapshot(context.Background())
+	if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 2 {
+		t.Fatalf("sizeByte = %+v, want 2 ints", f)
+	}
+	if _, ok := snap.Fields["title"]; !ok {
+		t.Fatalf("title missing: %v", snap.Fields)
+	}
+
+	if err := c.RebuildCounters(); err != nil {
+		t.Fatal(err)
+	}
+	if rebuilt := schemaJSON(t, c); rebuilt != incremental {
+		t.Fatalf("incremental counters drifted from a rebuild:\n incremental %s\n rebuilt     %s", incremental, rebuilt)
+	}
+	if n, _ := c.CountRoots(); n != 2 {
+		t.Fatalf("CountRoots after rebuild = %d", n)
+	}
+}
+
+func TestEnsureCountersBuildsOnce(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "One"}})
+	built, err := c.EnsureCounters()
+	if err != nil || !built {
+		t.Fatalf("first EnsureCounters built=%v err=%v", built, err)
+	}
+	if built, _ := c.EnsureCounters(); built {
+		t.Fatal("second EnsureCounters rebuilt again")
+	}
+}
+
+// A re-persisted record with nothing new writes nothing and publishes nothing.
+func TestUnchangedRecordWriteIsANoOp(t *testing.T) {
+	c := newTestClient(t)
+	rec := map[string]string{"title": "Naruto", "fileType": "video", "cids/bafkreiabc": "true"}
+	seed(t, c, map[string]map[string]string{"R1": rec})
+	before, _ := c.GetStreamLength(MetaEventsStream)
+	if err := c.SetMetadataFlat("R1", rec); err != nil {
+		t.Fatal(err)
+	}
+	if after, _ := c.GetStreamLength(MetaEventsStream); after != before {
+		t.Fatalf("re-persisting an unchanged record published %d events", after-before)
+	}
+}

internal/storage/search.go

@@ -211,21 +211,36 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
 	ctx, cancel := opCtx()
 	defer cancel()
 
-	driver, probes, err := c.planCandidates(ctx, s, query, filters)
+	plan, err := c.planCandidates(ctx, s, query, filters)
 	if err != nil {
 		return nil, err
 	}
 	page := max(limit*2, 500)
 	hits := make([]SearchHit, 0, limit)
 	after := ""
-	for len(hits) < limit {
-		ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
-		if err != nil {
-			return nil, err
+	for pos := 0; len(hits) < limit; {
+		var ids []string
+		if plan.walk {
+			rows, err := s.Range(ctx, rootsNS, after, page)
+			if err != nil {
+				return nil, err
+			}
+			for _, row := range rows {
+				ids = append(ids, row.K[len(rootsNS):])
+			}
+			if len(rows) > 0 {
+				after = rows[len(rows)-1].K
+			}
+		} else {
+			ids = plan.ordered[pos:min(pos+page, len(plan.ordered))]
+			pos += len(ids)
+		}
+		if len(ids) == 0 {
+			break
 		}
 		survivors := ids
-		if len(probes) > 0 && len(ids) > 0 {
-			if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
+		if len(plan.probes) > 0 {
+			if survivors, err = probeIDs(ctx, s, ids, plan.probes); err != nil {
 				return nil, err
 			}
 		}
@@ -245,30 +260,10 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
 		if len(ids) < page {
 			break
 		}
-		after = ids[len(ids)-1]
 	}
 	return hits, nil
 }
 
-// queryIDs returns one page of the driver's candidates, in rid order.
-func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
-	q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
-	rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
-	if err != nil {
-		return nil, fmt.Errorf("search candidates: %w", err)
-	}
-	defer rows.Close()
-	var out []string
-	for rows.Next() {
-		var id string
-		if err := rows.Scan(&id); err != nil {
-			return nil, err
-		}
-		out = append(out, id)
-	}
-	return out, rows.Err()
-}
-
 // probeSQL applies the non-driving filters to one page of candidate rids
 // (passed as a JSON array) — each an EXISTS over the candidate's own key range.
 func probeSQL(probes []candidatePart) (string, []any) {
@@ -304,7 +299,7 @@ func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidate
 	if err := rows.Err(); err != nil {
 		return nil, err
 	}
-	out := ids[:0:0]
+	out := make([]string, 0, len(ids))
 	for _, id := range ids {
 		if pass[id] {
 			out = append(out, id)
@@ -324,25 +319,40 @@ type candidatePart struct {
 	existsArgs []any
 }
 
-// estimateCap bounds the count used to pick the driving part: it only has to
-// tell a 50-row part from a 100k-row one.
-const estimateCap = 1000
+// estimateCap bounds the count used to rank parts by selectivity: past it, a
+// part is "broad" and only ties with other broad parts are left to resolve.
+var estimateCap = 50_000
 
-// planCandidates picks how to narrow a search: one DRIVING part (the most
-// selective, by a capped count) that lists candidate rids in order, plus PROBES
-// — every other filter, checked as an EXISTS over each candidate's own key
-// range (a primary-key range read of ~40 rows).
+// estimateCapForTest sets estimateCap and returns the previous value.
+func estimateCapForTest(n int) int { old := estimateCap; estimateCap = n; return old }
+
+// searchPlan is how one search walks its candidates.
+type searchPlan struct {
+	ordered []string        // the driver's rids, sorted (when !walk)
+	walk    bool            // no part narrows: walk idx:roots in order
+	probes  []candidatePart // filters checked per page as EXISTS
+}
+
+// planCandidates picks how to narrow a search.
 //
-// Search pages the driver in Go and probes one page at a time, so a broad
-// query stops as soon as `limit` records verify. Two earlier shapes were
-// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
-// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
-// and probing inside the driver query (SQLite evaluates the probe on every
-// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
-// both ~100k, 10 s). Free text that does not drive is dropped from SQL
-// entirely — the verifier still applies it, and dropping a condition only
-// widens the superset.
-func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
+// A capped count ranks the parts; the SMALLEST is collected once, sorted in Go
+// and paged from memory, and every other filter becomes a probe — an EXISTS
+// over each candidate's own key range, run one page at a time, so a search
+// stops as soon as `limit` records verify. The driver is collected in full even
+// when every part is broad: that costs O(part) — at most every root, ~0.3 s at
+// 169k — whereas walking idx:roots in hashID order (tried) costs O(corpus) when
+// matches are sparse early in that order, and on watch.nsl.sh they are: the
+// first 500 roots held zero episodes, so `contentKind:episode` walked ~3 s.
+// Only a search with nothing to narrow walks the roots, where every root
+// matches.
+//
+// Also rejected, measured on watch.nsl.sh: INTERSECTing fully materialised
+// sets (5.7 s for `naruto` + `fileType:video`); probing inside a driver query
+// re-sorted per page (SQLite evaluates the probe on every driver row before
+// ORDER BY … LIMIT can stop: 10 s for `1080p` + `fileType:video`). Free text
+// that does not drive is dropped from SQL — the verifier still applies it, and
+// dropping a condition only widens the superset.
+func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (searchPlan, error) {
 	var parts []candidatePart
 	if p, ok := textCandidates(query); ok {
 		parts = append(parts, p)
@@ -353,7 +363,7 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string,
 		}
 	}
 	if len(parts) == 0 {
-		return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
+		return searchPlan{walk: true}, nil
 	}
 
 	driver := 0
@@ -361,40 +371,67 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string,
 		best := int64(-1)
 		for i, p := range parts {
 			var n int64
-			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
+			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap+1) + `)`
 			if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
-				return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
+				return searchPlan{}, fmt.Errorf("estimate candidates: %w", err)
 			}
 			if best < 0 || n < best {
 				best, driver = n, i
 			}
+			if n == 0 {
+				break // nothing can match
+			}
 		}
 	}
-	var probes []candidatePart
+	ids, err := collectIDs(ctx, s, parts[driver], 0)
+	if err != nil {
+		return searchPlan{}, err
+	}
+	sort.Strings(ids)
+	plan := searchPlan{ordered: ids}
 	for i, p := range parts {
 		if i != driver && p.exists != "" {
-			probes = append(probes, p)
+			plan.probes = append(plan.probes, p)
 		}
 	}
-	return parts[driver], probes, nil
+	return plan, nil
 }
 
-// candidateSQL is planCandidates without the store: text plus filters, the
-// first part driving. Used by tests that inspect the plan.
+// collectIDs reads the distinct rids of one part (all of them when limit <= 0).
+func collectIDs(ctx context.Context, s *kv.Store, p candidatePart, limit int) ([]string, error) {
+	q := `SELECT DISTINCT rid FROM (` + p.sql + `)`
+	args := append([]any{}, p.args...)
+	if limit > 0 {
+		q += ` LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := s.DB().QueryContext(ctx, q, args...)
+	if err != nil {
+		return nil, fmt.Errorf("search candidates: %w", err)
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		out = append(out, id)
+	}
+	return out, rows.Err()
+}
+
+// candidateSQL is the first part's SQL, for tests that inspect the plan.
 func candidateSQL(query string, filters []SearchFilter) (string, []any) {
-	var parts []candidatePart
 	if p, ok := textCandidates(query); ok {
-		parts = append(parts, p)
+		return p.sql, p.args
 	}
 	for _, f := range filters {
 		if p, ok := filterCandidates(f); ok {
-			parts = append(parts, p)
+			return p.sql, p.args
 		}
 	}
-	if len(parts) == 0 {
-		return "", nil
-	}
-	return parts[0].sql, parts[0].args
+	return "", nil
 }
 
 // textCandidates narrows by free text: every whitespace token must be a
@@ -480,7 +517,7 @@ func filterCandidates(f SearchFilter) (candidatePart, bool) {
 	// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
 	// OR inside one SELECT would defeat the partial index; plain UNION would
 	// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
-	// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
+	// count over fileType:video on watch.nsl.sh). collectIDs applies DISTINCT.
 	base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
 	selects := make([]string, len(conds))
 	for i, cond := range conds {

internal/storage/search_test.go

@@ -294,13 +294,14 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
 	}
 	seed(t, c, recs)
 	filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
-	driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
+	plan, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 {
-		t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes))
+	if plan.walk || len(plan.ordered) != 1 || len(plan.probes) != 1 {
+		t.Fatalf("the rare text should drive (1 rid) with the filter as a probe, got %+v", plan)
 	}
+	probes := plan.probes
 	q, pargs := probeSQL(probes)
 	args := append([]any{`["R0000"]`}, pargs...)
 	rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
@@ -308,16 +309,16 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer rows.Close()
-	var plan []string
+	var steps []string
 	for rows.Next() {
 		var id, parent, notused int
 		var detail string
 		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
 			t.Fatal(err)
 		}
-		plan = append(plan, detail)
+		steps = append(steps, detail)
 	}
-	joined := strings.Join(plan, " | ")
+	joined := strings.Join(steps, " | ")
 	if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
 		t.Fatalf("filter probe is not a primary-key range read: %s", joined)
 	}
@@ -325,3 +326,30 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
 		return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
 	}), "R0000")
 }
+
+// When every part is broad (above estimateCap), the smallest is still
+// collected and drives — no walk over the roots — and the result is exact.
+func TestSearchBroadPartsStillDriveFromTheSmallest(t *testing.T) {
+	defer func(old int) { estimateCapForTest(old) }(estimateCapForTest(3))
+	c := newTestClient(t)
+	recs := map[string]map[string]string{}
+	for i := 0; i < 20; i++ {
+		ft := "video"
+		if i%2 == 1 {
+			ft = "audio"
+		}
+		recs[fmt.Sprintf("R%02d", i)] = map[string]string{"title": "Common words", "fileType": ft}
+	}
+	seed(t, c, recs)
+	filters := []SearchFilter{{Property: "fileType", Values: []string{"audio"}}}
+	plan, err := c.planCandidates(context.Background(), c.Store(), "common", filters)
+	if err != nil || plan.walk || len(plan.ordered) != 20 || len(plan.probes) != 1 {
+		t.Fatalf("want the text (20 rids) driving with one probe, got walk=%v ordered=%d probes=%d err %v", plan.walk, len(plan.ordered), len(plan.probes), err)
+	}
+	got := search(t, c, "common", filters, func(id string, f map[string]string) bool {
+		return textMatch("common")(id, f) && matchFilter(f, "fileType", false, "audio")
+	})
+	if len(got) != 10 || got[0] != "R01" {
+		t.Fatalf("got %v", got)
+	}
+}

internal/storage/stats.go

@@ -4,8 +4,9 @@ import (
 	"time"
 )
 
-// CountRoots returns the number of metadata roots in the root index — one
-// range count over idx:roots/, never the list itself.
+// CountRoots returns the number of metadata roots: a counter the write path
+// keeps exact (see onChange), so it costs one read — search reports it as
+// `total` on every query.
 func (c *Client) CountRoots() (int64, error) {
 	s, err := c.kv()
 	if err != nil {
@@ -13,7 +14,8 @@ func (c *Client) CountRoots() (int64, error) {
 	}
 	ctx, cancel := opCtx()
 	defer cancel()
-	return s.Count(ctx, rootsNS)
+	n, _, err := s.Counter(ctx, rootsCountKey)
+	return n, err
 }
 
 // keyCountTTL bounds how stale CountKeys may be. Counting every row is a full

internal/storage/write_bench_test.go

@@ -57,3 +57,53 @@ func BenchmarkConcurrentRecordWrites(b *testing.B) {
 	wg.Wait()
 	b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
 }
+
+// BenchmarkConcurrentRepersistUnchanged is the dominant write on a permanent
+// store: a gateway re-persisting hits it already holds. The kv layer skips
+// every unchanged field, so this should cost reads only.
+func BenchmarkConcurrentRepersistUnchanged(b *testing.B) {
+	c := NewClient()
+	if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
+		b.Fatal(err)
+	}
+	defer c.Close()
+	const distinct = 2000
+	rec := func(i int64) map[string]string {
+		m := map[string]string{
+			"title":    fmt.Sprintf("Some Show S01E%02d", i%40),
+			"fileType": "video",
+			fmt.Sprintf("cids/bafkbench%020d", i): "true",
+		}
+		for f := 0; len(m) < 40; f++ {
+			m[fmt.Sprintf("categories/%d", f)] = "true"
+		}
+		return m
+	}
+	for i := int64(0); i < distinct; i++ {
+		if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), rec(i)); err != nil {
+			b.Fatal(err)
+		}
+	}
+	const writers = 16
+	var next int64
+	b.ResetTimer()
+	var wg sync.WaitGroup
+	for w := 0; w < writers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				i := atomic.AddInt64(&next, 1)
+				if i > int64(b.N) {
+					return
+				}
+				if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i%distinct), rec(i%distinct)); err != nil {
+					b.Error(err)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
+}