meta-core/internal/schema/indexer.go

package schema

import (
	"context"
	"log"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/metazla/meta-core/internal/kv"
)

var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)

// rebuildInterval is how often the schema is recomputed in the background.
// It describes the *shape* of the data (which fields exist, their types), which
// drifts slowly; a few minutes of staleness is invisible to the editor that
// reads it, and POST /api/schema/rescan recomputes on demand.
const rebuildInterval = 10 * time.Minute

// fieldState is the per-field accumulator.
type fieldState struct {
	counts         map[string]int
	primitives     map[Primitive]int
	hintCounts     map[Hint]int
	typedCount     int
	undefinedCount int
	languageKeyed  bool
}

func newFieldState() *fieldState {
	return &fieldState{
		counts:     make(map[string]int),
		primitives: make(map[Primitive]int),
		hintCounts: make(map[Hint]int),
	}
}

// Indexer maintains the per-field schema by scanning the store.
//
// It used to follow meta:events and keep, per field, a map of EVERY record key
// it had seen (so a re-set or delete could undo that key's contribution). At
// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident
// memory once the search copy was gone — and it only worked if every event was
// replayed into it, which is half of why meta-core republished every field on
// boot. A periodic streaming scan holds one accumulator per FIELD instead and
// is exact by construction.
type Indexer struct {
	store *kv.Store

	mu   sync.RWMutex
	snap *SchemaResponse

	rebuildMu sync.Mutex

	ctx     context.Context
	cancel  context.CancelFunc
	wg      sync.WaitGroup
	runMu   sync.Mutex
	running bool
}

// NewIndexer creates an Indexer over store.
func NewIndexer(store *kv.Store) *Indexer {
	ctx, cancel := context.WithCancel(context.Background())
	return &Indexer{store: store, ctx: ctx, cancel: cancel}
}

// Start computes the schema in the background now and every rebuildInterval.
func (i *Indexer) Start() error {
	i.runMu.Lock()
	defer i.runMu.Unlock()
	if i.running {
		return nil
	}
	i.running = true
	i.wg.Add(1)
	go func() {
		defer i.wg.Done()
		ticker := time.NewTicker(rebuildInterval)
		defer ticker.Stop()
		for {
			if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil {
				log.Printf("[SchemaIndexer] rebuild failed: %v", err)
			}
			select {
			case <-i.ctx.Done():
				return
			case <-ticker.C:
			}
		}
	}()
	log.Println("[SchemaIndexer] Started - scanning the store")
	return nil
}

// Stop terminates the background rebuilds.
func (i *Indexer) Stop() error {
	i.runMu.Lock()
	if !i.running {
		i.runMu.Unlock()
		return nil
	}
	i.running = false
	i.runMu.Unlock()

	i.cancel()
	i.wg.Wait()
	log.Println("[SchemaIndexer] Stopped")
	return nil
}

// Rebuild recomputes the schema from every record key and publishes it. One
// rebuild runs at a time; a concurrent caller waits and gets the fresh result.
func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) {
	i.rebuildMu.Lock()
	defer i.rebuildMu.Unlock()

	started := time.Now()
	fields := make(map[string]*fieldState)
	err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
		for _, row := range rows {
			fieldPath, keyHint, ok := parseFieldPath(row.K)
			if !ok {
				continue
			}
			fs := fields[fieldPath]
			if fs == nil {
				fs = newFieldState()
				fields[fieldPath] = fs
			}
			if keyHint != "" {
				fs.languageKeyed = true
			}
			prim, hint := ClassifyValue(row.V, true)
			incContribution(fs, labelPair{prim: prim, hint: hint})
		}
		return ctx.Err()
	})
	if err != nil {
		return nil, err
	}

	snap := &SchemaResponse{
		Fields:      make(map[string]*FieldSchema, len(fields)),
		GeneratedAt: time.Now().UTC(),
		Source:      "scan",
	}
	for fp, fs := range fields {
		snap.Fields[fp] = computeFieldSchema(fs)
	}
	i.mu.Lock()
	i.snap = snap
	i.mu.Unlock()
	log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond))
	return snap, nil
}

type labelPair struct {
	prim Primitive
	hint Hint
}

// parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
// path used as the map key. rootID is opaque to the indexer — currently a
// UUIDv7, historically a midhash256:abc token; both shapes parse identically
// because the split is on the first '/' after "file:".
//
// When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as
// a language code: the field name collapses to the prefix and key_hint is set.
func parseFieldPath(key string) (string, KeyHint, bool) {
	if !strings.HasPrefix(key, "file:") {
		return "", "", false
	}
	rest := strings.TrimPrefix(key, "file:")
	slash := strings.Index(rest, "/")
	if slash < 0 {
		return "", "", false
	}
	property := rest[slash+1:]
	if property == "" {
		return "", "", false
	}
	if last := strings.LastIndex(property, "/"); last >= 0 {
		tail := property[last+1:]
		if langCodeRegex.MatchString(tail) {
			return property[:last], KeyHintLanguageCode, true
		}
	}
	return property, "", true
}

func incContribution(fs *fieldState, lp labelPair) {
	label := breakdownLabel(lp.prim, lp.hint)
	fs.counts[label]++
	if lp.prim == PrimUndefined {
		fs.undefinedCount++
		return
	}
	fs.typedCount++
	fs.primitives[lp.prim]++
	if lp.hint != "" {
		fs.hintCounts[lp.hint]++
	}
}

// Snapshot returns the latest computed schema. Before the first scan finishes
// it is empty with Source "pending".
func (i *Indexer) Snapshot() *SchemaResponse {
	i.mu.RLock()
	defer i.mu.RUnlock()
	if i.snap == nil {
		return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"}
	}
	return i.snap
}

func computeFieldSchema(fs *fieldState) *FieldSchema {
	out := &FieldSchema{
		Breakdown: make(map[string]int, len(fs.counts)),
	}
	for k, v := range fs.counts {
		out.Breakdown[k] = v
	}

	nPrim := len(fs.primitives)
	hasUndef := fs.undefinedCount > 0

	switch {
	case nPrim == 0 && hasUndef:
		out.Type = PrimUndefined
	case nPrim == 1 && !hasUndef:
		for p := range fs.primitives {
			out.Type = p
		}
	default:
		out.Type = PrimMixed
	}

	// Strict hint promotion: exactly one hint observed, and it covers every
	// typed (non-undefined) value. Hints are never set on mixed fields.
	if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 {
		for h, c := range fs.hintCounts {
			if c == fs.typedCount {
				out.Hint = h
			}
		}
	}

	if fs.languageKeyed {
		out.KeyHint = KeyHintLanguageCode
	}

	return out
}