v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/schema/indexer.go

@@ -2,10 +2,9 @@ package schema
 
 import (
 	"context"
-	"log"
 	"regexp"
+	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/metazla/meta-core/internal/kv"
@@ -13,11 +12,23 @@ import (
 
 var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
 
-// rebuildInterval is how often the schema is recomputed in the background.
-// It describes the *shape* of the data (which fields exist, their types), which
-// drifts slowly; a few minutes of staleness is invisible to the editor that
-// reads it, and POST /api/schema/rescan recomputes on demand.
-const rebuildInterval = 10 * time.Minute
+// CounterPrefix is where the schema lives: one counter per (field, key hint,
+// primitive, hint), maintained by the storage write path in the same
+// transaction as the change (storage's OnChange hook → CounterKey).
+const CounterPrefix = "idx:schema/"
+
+const counterSep = "\x1f"
+
+// CounterKey is the counter a record key holding value contributes to, or
+// ok=false for keys that are not record properties.
+func CounterKey(recordKey, value string) (string, bool) {
+	fieldPath, keyHint, ok := parseFieldPath(recordKey)
+	if !ok {
+		return "", false
+	}
+	prim, hint := ClassifyValue(value, true)
+	return CounterPrefix + fieldPath + counterSep + string(keyHint) + counterSep + string(prim) + counterSep + string(hint), true
+}
 
 // fieldState is the per-field accumulator.
 type fieldState struct {
@@ -37,123 +48,63 @@ func newFieldState() *fieldState {
 	}
 }
 
-// Indexer maintains the per-field schema by scanning the store.
+// Indexer serves the per-field schema.
 //
-// It used to follow meta:events and keep, per field, a map of EVERY record key
-// it had seen (so a re-set or delete could undo that key's contribution). At
-// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident
-// memory once the search copy was gone — and it only worked if every event was
-// replayed into it, which is half of why meta-core republished every field on
-// boot. A periodic streaming scan holds one accumulator per FIELD instead and
-// is exact by construction.
+// The schema is a set of exact counters the write path keeps current in the
+// same transaction as each change (see CounterKey), so serving it is one range
+// read and nothing runs in the background. Two earlier shapes were dropped: a
+// consumer of meta:events that remembered every record key it had seen (~2 GB
+// at 6.3M keys), then a full rescan of the store every 10 minutes (~80 s of
+// CPU each time, whether or not anyone looked).
 type Indexer struct {
 	store *kv.Store
-
-	mu   sync.RWMutex
-	snap *SchemaResponse
-
-	rebuildMu sync.Mutex
-
-	ctx     context.Context
-	cancel  context.CancelFunc
-	wg      sync.WaitGroup
-	runMu   sync.Mutex
-	running bool
 }
 
 // NewIndexer creates an Indexer over store.
 func NewIndexer(store *kv.Store) *Indexer {
-	ctx, cancel := context.WithCancel(context.Background())
-	return &Indexer{store: store, ctx: ctx, cancel: cancel}
+	return &Indexer{store: store}
 }
 
-// Start computes the schema in the background now and every rebuildInterval.
-func (i *Indexer) Start() error {
-	i.runMu.Lock()
-	defer i.runMu.Unlock()
-	if i.running {
-		return nil
-	}
-	i.running = true
-	i.wg.Add(1)
-	go func() {
-		defer i.wg.Done()
-		ticker := time.NewTicker(rebuildInterval)
-		defer ticker.Stop()
-		for {
-			if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil {
-				log.Printf("[SchemaIndexer] rebuild failed: %v", err)
-			}
-			select {
-			case <-i.ctx.Done():
-				return
-			case <-ticker.C:
-			}
-		}
-	}()
-	log.Println("[SchemaIndexer] Started - scanning the store")
-	return nil
-}
+// Start and Stop exist for the server's lifecycle; there is nothing to run.
+func (i *Indexer) Start() error { return nil }
+func (i *Indexer) Stop() error  { return nil }
 
-// Stop terminates the background rebuilds.
-func (i *Indexer) Stop() error {
-	i.runMu.Lock()
-	if !i.running {
-		i.runMu.Unlock()
-		return nil
-	}
-	i.running = false
-	i.runMu.Unlock()
-
-	i.cancel()
-	i.wg.Wait()
-	log.Println("[SchemaIndexer] Stopped")
-	return nil
-}
-
-// Rebuild recomputes the schema from every record key and publishes it. One
-// rebuild runs at a time; a concurrent caller waits and gets the fresh result.
-func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) {
-	i.rebuildMu.Lock()
-	defer i.rebuildMu.Unlock()
-
-	started := time.Now()
+// Snapshot reads the counters and folds them into the per-field schema.
+func (i *Indexer) Snapshot(ctx context.Context) (*SchemaResponse, error) {
 	fields := make(map[string]*fieldState)
-	err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
+	err := i.store.ScanPrefix(ctx, CounterPrefix, 5000, func(rows []kv.KV) error {
 		for _, row := range rows {
-			fieldPath, keyHint, ok := parseFieldPath(row.K)
-			if !ok {
+			parts := strings.Split(row.K[len(CounterPrefix):], counterSep)
+			if len(parts) != 4 {
 				continue
 			}
-			fs := fields[fieldPath]
+			n, err := strconv.Atoi(row.V)
+			if err != nil || n <= 0 {
+				continue
+			}
+			fs := fields[parts[0]]
 			if fs == nil {
 				fs = newFieldState()
-				fields[fieldPath] = fs
+				fields[parts[0]] = fs
 			}
-			if keyHint != "" {
+			if parts[1] != "" {
 				fs.languageKeyed = true
 			}
-			prim, hint := ClassifyValue(row.V, true)
-			incContribution(fs, labelPair{prim: prim, hint: hint})
+			incContribution(fs, labelPair{prim: Primitive(parts[2]), hint: Hint(parts[3])}, n)
 		}
-		return ctx.Err()
+		return nil
 	})
 	if err != nil {
 		return nil, err
 	}
-
 	snap := &SchemaResponse{
 		Fields:      make(map[string]*FieldSchema, len(fields)),
 		GeneratedAt: time.Now().UTC(),
-		Source:      "scan",
+		Source:      "live",
 	}
 	for fp, fs := range fields {
 		snap.Fields[fp] = computeFieldSchema(fs)
 	}
-	i.mu.Lock()
-	i.snap = snap
-	i.mu.Unlock()
-	log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond))
 	return snap, nil
 }
 
@@ -191,31 +142,21 @@ func parseFieldPath(key string) (string, KeyHint, bool) {
 	return property, "", true
 }
 
-func incContribution(fs *fieldState, lp labelPair) {
+// incContribution adds n values of one (primitive, hint) to a field.
+func incContribution(fs *fieldState, lp labelPair, n int) {
 	label := breakdownLabel(lp.prim, lp.hint)
-	fs.counts[label]++
+	fs.counts[label] += n
 	if lp.prim == PrimUndefined {
-		fs.undefinedCount++
+		fs.undefinedCount += n
 		return
 	}
-	fs.typedCount++
-	fs.primitives[lp.prim]++
+	fs.typedCount += n
+	fs.primitives[lp.prim] += n
 	if lp.hint != "" {
-		fs.hintCounts[lp.hint]++
+		fs.hintCounts[lp.hint] += n
 	}
 }
 
-// Snapshot returns the latest computed schema. Before the first scan finishes
-// it is empty with Source "pending".
-func (i *Indexer) Snapshot() *SchemaResponse {
-	i.mu.RLock()
-	defer i.mu.RUnlock()
-	if i.snap == nil {
-		return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"}
-	}
-	return i.snap
-}
-
 func computeFieldSchema(fs *fieldState) *FieldSchema {
 	out := &FieldSchema{
 		Breakdown: make(map[string]int, len(fs.counts)),

internal/schema/indexer_test.go

@@ -1,56 +0,0 @@
-package schema
-
-import (
-	"context"
-	"path/filepath"
-	"testing"
-
-	"github.com/metazla/meta-core/internal/kv"
-)
-
-func TestRebuildDerivesTypesFromTheStore(t *testing.T) {
-	store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer store.Close()
-	if err := store.Update(context.Background(), func(tx *kv.Tx) error {
-		for k, v := range map[string]string{
-			"file:A/sizeByte":   "100",
-			"file:B/sizeByte":   "200",
-			"file:A/title/eng":  "One",
-			"file:B/title/fre":  "Deux",
-			"file:A/mixed":      "1",
-			"file:B/mixed":      "x",
-			"cid:bafknotrecord": "A",
-		} {
-			if err := tx.Put(k, v); err != nil {
-				return err
-			}
-		}
-		return nil
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	ix := NewIndexer(store)
-	if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 {
-		t.Fatalf("before a scan: %+v", got)
-	}
-	snap, err := ix.Rebuild(context.Background())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 {
-		t.Fatalf("sizeByte = %+v", f)
-	}
-	if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode {
-		t.Fatalf("title = %+v (want language-keyed)", f)
-	}
-	if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed {
-		t.Fatalf("mixed = %+v", f)
-	}
-	if len(snap.Fields) != 3 {
-		t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields)
-	}
-}