v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/storage/client.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/metazla/meta-core/internal/kv"
+	"github.com/metazla/meta-core/internal/schema"
 )
 
 // Key layout (everything is one string→string table, see internal/kv):
@@ -21,6 +22,7 @@ import (
 //	cid:<cid>                reverse index → <rid>
 //	idx:roots/<rid>          the set of every root (was the Redis SET file:__index__)
 //	idx:fts/<rid>            rowid of the record's row in the FTS table
+//	idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
 //	dup:<rid>/<path>         a duplicate path of the record (was the SET file:<rid>/duplicates)
 //	udl:…                    User Data Layer, see udl.go
 //	ev:<stream>/…, evhwm:…   event streams, see internal/kv/stream.go
@@ -80,6 +82,7 @@ func (c *Client) Connect(path string) error {
 		ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
 		Readers:       envInt("META_CORE_DB_READERS", 8),
 		Schema:        searchSchema,
+		OnChange:      onChange,
 	})
 	if err != nil {
 		return fmt.Errorf("open %s: %w", path, err)
@@ -185,6 +188,101 @@ func isInternalKey(key string) bool {
 		strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS)
 }
 
+// Derived counters, maintained by onChange in the same transaction as the
+// change they count (kv.Tx.AddCounter): the root count behind search's `total`
+// and /api/stats, and the per-field schema behind /api/schema.
+const (
+	rootsCountKey  = "idx:count/roots"
+	countersReady  = "idx:count/ready"
+	counterRebuild = 5000
+)
+
+// onChange keeps the derived counters exact. It sees only real changes (kv
+// skips unchanged Puts) with the old and new values, so a modified field moves
+// one count from its old label to its new one.
+func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) {
+	if strings.HasPrefix(k, rootsNS) {
+		switch {
+		case hasNew && !hadOld:
+			tx.AddCounter(rootsCountKey, 1)
+		case hadOld && !hasNew:
+			tx.AddCounter(rootsCountKey, -1)
+		}
+		return
+	}
+	if !strings.HasPrefix(k, recordNS) {
+		return
+	}
+	if hadOld {
+		if ck, ok := schema.CounterKey(k, old); ok {
+			tx.AddCounter(ck, -1)
+		}
+	}
+	if hasNew {
+		if ck, ok := schema.CounterKey(k, v); ok {
+			tx.AddCounter(ck, 1)
+		}
+	}
+}
+
+// EnsureCounters builds the derived counters when they have never been built —
+// a store created before they existed, or one restored from outside meta-core.
+// Blocking and single-transaction (writes wait for it; ~1–2 min at millions of
+// keys, once), because a build racing live deltas would double-count.
+func (c *Client) EnsureCounters() (bool, error) {
+	s, err := c.kv()
+	if err != nil {
+		return false, err
+	}
+	ctx, cancel := opCtx()
+	_, ready, err := s.Get(ctx, countersReady)
+	cancel()
+	if err != nil || ready {
+		return false, err
+	}
+	return true, c.RebuildCounters()
+}
+
+// RebuildCounters recomputes every derived counter from the records, in one
+// transaction (writers wait for it).
+func (c *Client) RebuildCounters() error {
+	s, err := c.kv()
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	return s.Update(ctx, func(tx *kv.Tx) error {
+		for _, p := range []string{schema.CounterPrefix, rootsCountKey} {
+			if _, err := tx.DeletePrefix(p, 0); err != nil {
+				return err
+			}
+		}
+		roots, err := tx.Count(tx.Context(), rootsNS)
+		if err != nil {
+			return err
+		}
+		tx.AddCounter(rootsCountKey, roots)
+		after := ""
+		for {
+			rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild)
+			if err != nil {
+				return err
+			}
+			for _, row := range rows {
+				if ck, ok := schema.CounterKey(row.K, row.V); ok {
+					tx.AddCounter(ck, 1)
+				}
+			}
+			if len(rows) < counterRebuild {
+				break
+			}
+			after = rows[len(rows)-1].K
+		}
+		return tx.Put(countersReady, "1")
+	})
+}
+
 // readRecord returns a record's flat fields (nil when it has none).
 func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) {
 	prefix := recordPrefix(hashID)
@@ -210,17 +308,19 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
 		return fmt.Errorf("empty hash id")
 	}
 	prefix := recordPrefix(hashID)
-	touchesSearch := false
+	searchChanged := false
 	for field, value := range metadata {
 		if isReservedFieldName(field) {
 			continue
 		}
-		if err := tx.Put(prefix+field, value); err != nil {
+		changed, err := tx.Set(prefix+field, value)
+		if err != nil {
 			return err
 		}
-		touchesSearch = touchesSearch || affectsHaystack(field)
+		searchChanged = searchChanged || (changed && affectsHaystack(field))
 	}
-	if err := tx.Put(rootKey(hashID), ""); err != nil {
+	newRoot, err := tx.Set(rootKey(hashID), "")
+	if err != nil {
 		return err
 	}
 	for field := range metadata {
@@ -230,10 +330,12 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
 			}
 		}
 	}
-	if touchesSearch {
+	// The FTS row is rewritten only when the haystack can have changed — a
+	// re-persisted record whose values are all in place costs reads only.
+	if searchChanged || newRoot {
 		return refreshSearchTx(tx, hashID)
 	}
-	return ensureSearchRowTx(tx, hashID)
+	return nil
 }
 
 // GetMetadataFlat retrieves all metadata for a file as a flat map.