replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/schema/classify.go

@@ -21,7 +21,7 @@ import (
 // (crc32: "bafk3eaqerjhiwhy").
 var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
 
-// ClassifyValue returns the (primitive, hint) for a Redis value.
+// ClassifyValue returns the (primitive, hint) for a stored value.
 // exists=false (key missing) and JSON null literal both map to PrimUndefined.
 // Empty string is a valid string.
 func ClassifyValue(value string, exists bool) (Primitive, Hint) {

internal/schema/indexer.go

@@ -6,17 +6,20 @@ import (
 	"regexp"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/metazla/meta-core/internal/events"
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
 
-// fieldState is the per-field accumulator. perKey lets us undo a previous
-// contribution when the same Redis key is re-SET or DELed.
+// rebuildInterval is how often the schema is recomputed in the background.
+// It describes the *shape* of the data (which fields exist, their types), which
+// drifts slowly; a few minutes of staleness is invisible to the editor that
+// reads it, and POST /api/schema/rescan recomputes on demand.
+const rebuildInterval = 10 * time.Minute
+
+// fieldState is the per-field accumulator.
 type fieldState struct {
 	counts         map[string]int
 	primitives     map[Primitive]int
@@ -24,12 +27,6 @@ type fieldState struct {
 	typedCount     int
 	undefinedCount int
 	languageKeyed  bool
-	perKey         map[string]labelPair
-}
-
-type labelPair struct {
-	prim Primitive
-	hint Hint
 }
 
 func newFieldState() *fieldState {
@@ -37,19 +34,25 @@ func newFieldState() *fieldState {
 		counts:     make(map[string]int),
 		primitives: make(map[Primitive]int),
 		hintCounts: make(map[Hint]int),
-		perKey:     make(map[string]labelPair),
 	}
 }
 
-// Indexer maintains a live field-schema by consuming meta:events.
+// Indexer maintains the per-field schema by scanning the store.
+//
+// It used to follow meta:events and keep, per field, a map of EVERY record key
+// it had seen (so a re-set or delete could undo that key's contribution). At
+// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident
+// memory once the search copy was gone — and it only worked if every event was
+// replayed into it, which is half of why meta-core republished every field on
+// boot. A periodic streaming scan holds one accumulator per FIELD instead and
+// is exact by construction.
 type Indexer struct {
-	client    *redis.Client
-	keyPrefix string
+	store *kv.Store
 
-	mu     sync.RWMutex
-	fields map[string]*fieldState
+	mu   sync.RWMutex
+	snap *SchemaResponse
 
-	processed uint64 // total events processed since start
+	rebuildMu sync.Mutex
 
 	ctx     context.Context
 	cancel  context.CancelFunc
@@ -58,37 +61,41 @@ type Indexer struct {
 	running bool
 }
 
-// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
-// so the indexer can build absolute Redis keys for GET calls.
-func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
+// NewIndexer creates an Indexer over store.
+func NewIndexer(store *kv.Store) *Indexer {
 	ctx, cancel := context.WithCancel(context.Background())
-	return &Indexer{
-		client:    client,
-		keyPrefix: keyPrefix,
-		fields:    make(map[string]*fieldState),
-		ctx:       ctx,
-		cancel:    cancel,
-	}
+	return &Indexer{store: store, ctx: ctx, cancel: cancel}
 }
 
-// Start begins consuming the meta:events stream from the earliest available ID.
+// Start computes the schema in the background now and every rebuildInterval.
 func (i *Indexer) Start() error {
 	i.runMu.Lock()
+	defer i.runMu.Unlock()
 	if i.running {
-		i.runMu.Unlock()
 		return nil
 	}
 	i.running = true
-	i.runMu.Unlock()
-
 	i.wg.Add(1)
-	go i.consumeLoop()
-
-	log.Println("[SchemaIndexer] Started - consuming meta:events stream")
+	go func() {
+		defer i.wg.Done()
+		ticker := time.NewTicker(rebuildInterval)
+		defer ticker.Stop()
+		for {
+			if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil {
+				log.Printf("[SchemaIndexer] rebuild failed: %v", err)
+			}
+			select {
+			case <-i.ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	log.Println("[SchemaIndexer] Started - scanning the store")
 	return nil
 }
 
-// Stop terminates the consumer goroutine.
+// Stop terminates the background rebuilds.
 func (i *Indexer) Stop() error {
 	i.runMu.Lock()
 	if !i.running {
@@ -104,93 +111,55 @@ func (i *Indexer) Stop() error {
 	return nil
 }
 
-// ProcessedCount returns the total number of stream events processed.
-// Used by WaitForDrain to detect when a rescan flood has been absorbed.
-func (i *Indexer) ProcessedCount() uint64 {
-	return atomic.LoadUint64(&i.processed)
-}
+// Rebuild recomputes the schema from every record key and publishes it. One
+// rebuild runs at a time; a concurrent caller waits and gets the fresh result.
+func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) {
+	i.rebuildMu.Lock()
+	defer i.rebuildMu.Unlock()
 
-// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
-// the timeout elapses. Suitable for use right after RepublishMetadata.
-func (i *Indexer) WaitForDrain(timeout time.Duration) {
-	deadline := time.Now().Add(timeout)
-	last := i.ProcessedCount()
-	stableSince := time.Now()
-	for time.Now().Before(deadline) {
-		time.Sleep(100 * time.Millisecond)
-		now := i.ProcessedCount()
-		if now != last {
-			last = now
-			stableSince = time.Now()
-			continue
-		}
-		if time.Since(stableSince) >= 300*time.Millisecond {
-			return
-		}
-	}
-}
-
-func (i *Indexer) consumeLoop() {
-	defer i.wg.Done()
-	lastID := "0"
-	for {
-		select {
-		case <-i.ctx.Done():
-			return
-		default:
-		}
-
-		res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
-			Streams: []string{events.MetaEventsStream, lastID},
-			Block:   2 * time.Second,
-			Count:   500,
-		}).Result()
-
-		if err != nil {
-			if err == redis.Nil || i.ctx.Err() != nil {
+	started := time.Now()
+	fields := make(map[string]*fieldState)
+	err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
+		for _, row := range rows {
+			fieldPath, keyHint, ok := parseFieldPath(row.K)
+			if !ok {
 				continue
 			}
-			log.Printf("[SchemaIndexer] XRead error: %v", err)
-			time.Sleep(1 * time.Second)
-			continue
-		}
-
-		for _, stream := range res {
-			for _, msg := range stream.Messages {
-				lastID = msg.ID
-				i.processEvent(msg.Values)
-				atomic.AddUint64(&i.processed, 1)
+			fs := fields[fieldPath]
+			if fs == nil {
+				fs = newFieldState()
+				fields[fieldPath] = fs
 			}
+			if keyHint != "" {
+				fs.languageKeyed = true
+			}
+			prim, hint := ClassifyValue(row.V, true)
+			incContribution(fs, labelPair{prim: prim, hint: hint})
 		}
+		return ctx.Err()
+	})
+	if err != nil {
+		return nil, err
 	}
+
+	snap := &SchemaResponse{
+		Fields:      make(map[string]*FieldSchema, len(fields)),
+		GeneratedAt: time.Now().UTC(),
+		Source:      "scan",
+	}
+	for fp, fs := range fields {
+		snap.Fields[fp] = computeFieldSchema(fs)
+	}
+	i.mu.Lock()
+	i.snap = snap
+	i.mu.Unlock()
+	log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond))
+	return snap, nil
 }
 
-func (i *Indexer) processEvent(values map[string]any) {
-	op, _ := values["type"].(string)
-	key, _ := values["key"].(string)
-	if key == "" {
-		return
-	}
-	fieldPath, keyHint, ok := parseFieldPath(key)
-	if !ok {
-		return
-	}
-
-	switch op {
-	case "set":
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
-		cancel()
-		exists := err != redis.Nil
-		if err != nil && err != redis.Nil {
-			// Treat transient errors as "value disappeared" — nothing to record.
-			return
-		}
-		prim, hint := ClassifyValue(val, exists)
-		i.upsert(fieldPath, key, keyHint, prim, hint)
-	case "del", "expire":
-		i.remove(fieldPath, key)
-	}
+type labelPair struct {
+	prim Primitive
+	hint Hint
 }
 
 // parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
@@ -222,46 +191,6 @@ func parseFieldPath(key string) (string, KeyHint, bool) {
 	return property, "", true
 }
 
-func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		fs = newFieldState()
-		i.fields[fieldPath] = fs
-	}
-	if keyHint != "" {
-		fs.languageKeyed = true
-	}
-
-	if old, ok := fs.perKey[redisKey]; ok {
-		decContribution(fs, old)
-	}
-
-	fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
-	incContribution(fs, labelPair{prim: prim, hint: hint})
-}
-
-func (i *Indexer) remove(fieldPath, redisKey string) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		return
-	}
-	old, ok := fs.perKey[redisKey]
-	if !ok {
-		return
-	}
-	delete(fs.perKey, redisKey)
-	decContribution(fs, old)
-	if len(fs.perKey) == 0 {
-		delete(i.fields, fieldPath)
-	}
-}
-
 func incContribution(fs *fieldState, lp labelPair) {
 	label := breakdownLabel(lp.prim, lp.hint)
 	fs.counts[label]++
@@ -276,43 +205,15 @@ func incContribution(fs *fieldState, lp labelPair) {
 	}
 }
 
-func decContribution(fs *fieldState, lp labelPair) {
-	label := breakdownLabel(lp.prim, lp.hint)
-	fs.counts[label]--
-	if fs.counts[label] <= 0 {
-		delete(fs.counts, label)
-	}
-	if lp.prim == PrimUndefined {
-		fs.undefinedCount--
-		return
-	}
-	fs.typedCount--
-	fs.primitives[lp.prim]--
-	if fs.primitives[lp.prim] <= 0 {
-		delete(fs.primitives, lp.prim)
-	}
-	if lp.hint != "" {
-		fs.hintCounts[lp.hint]--
-		if fs.hintCounts[lp.hint] <= 0 {
-			delete(fs.hintCounts, lp.hint)
-		}
-	}
-}
-
-// Snapshot returns the current schema. Safe to call concurrently with consumption.
+// Snapshot returns the latest computed schema. Before the first scan finishes
+// it is empty with Source "pending".
 func (i *Indexer) Snapshot() *SchemaResponse {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
-
-	out := &SchemaResponse{
-		Fields:      make(map[string]*FieldSchema, len(i.fields)),
-		GeneratedAt: time.Now().UTC(),
-		Source:      "live",
+	if i.snap == nil {
+		return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"}
 	}
-	for fp, fs := range i.fields {
-		out.Fields[fp] = computeFieldSchema(fs)
-	}
-	return out
+	return i.snap
 }
 
 func computeFieldSchema(fs *fieldState) *FieldSchema {

internal/schema/indexer_test.go

@@ -0,0 +1,56 @@
+package schema
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+func TestRebuildDerivesTypesFromTheStore(t *testing.T) {
+	store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+	if err := store.Update(context.Background(), func(tx *kv.Tx) error {
+		for k, v := range map[string]string{
+			"file:A/sizeByte":   "100",
+			"file:B/sizeByte":   "200",
+			"file:A/title/eng":  "One",
+			"file:B/title/fre":  "Deux",
+			"file:A/mixed":      "1",
+			"file:B/mixed":      "x",
+			"cid:bafknotrecord": "A",
+		} {
+			if err := tx.Put(k, v); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	ix := NewIndexer(store)
+	if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 {
+		t.Fatalf("before a scan: %+v", got)
+	}
+	snap, err := ix.Rebuild(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 {
+		t.Fatalf("sizeByte = %+v", f)
+	}
+	if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode {
+		t.Fatalf("title = %+v (want language-keyed)", f)
+	}
+	if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed {
+		t.Fatalf("mixed = %+v", f)
+	}
+	if len(snap.Fields) != 3 {
+		t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields)
+	}
+}