package schema import ( "context" "log" "regexp" "strings" "sync" "time" "github.com/metazla/meta-core/internal/kv" ) var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`) // rebuildInterval is how often the schema is recomputed in the background. // It describes the *shape* of the data (which fields exist, their types), which // drifts slowly; a few minutes of staleness is invisible to the editor that // reads it, and POST /api/schema/rescan recomputes on demand. const rebuildInterval = 10 * time.Minute // fieldState is the per-field accumulator. type fieldState struct { counts map[string]int primitives map[Primitive]int hintCounts map[Hint]int typedCount int undefinedCount int languageKeyed bool } func newFieldState() *fieldState { return &fieldState{ counts: make(map[string]int), primitives: make(map[Primitive]int), hintCounts: make(map[Hint]int), } } // Indexer maintains the per-field schema by scanning the store. // // It used to follow meta:events and keep, per field, a map of EVERY record key // it had seen (so a re-set or delete could undo that key's contribution). At // 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident // memory once the search copy was gone — and it only worked if every event was // replayed into it, which is half of why meta-core republished every field on // boot. A periodic streaming scan holds one accumulator per FIELD instead and // is exact by construction. type Indexer struct { store *kv.Store mu sync.RWMutex snap *SchemaResponse rebuildMu sync.Mutex ctx context.Context cancel context.CancelFunc wg sync.WaitGroup runMu sync.Mutex running bool } // NewIndexer creates an Indexer over store. func NewIndexer(store *kv.Store) *Indexer { ctx, cancel := context.WithCancel(context.Background()) return &Indexer{store: store, ctx: ctx, cancel: cancel} } // Start computes the schema in the background now and every rebuildInterval. func (i *Indexer) Start() error { i.runMu.Lock() defer i.runMu.Unlock() if i.running { return nil } i.running = true i.wg.Add(1) go func() { defer i.wg.Done() ticker := time.NewTicker(rebuildInterval) defer ticker.Stop() for { if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil { log.Printf("[SchemaIndexer] rebuild failed: %v", err) } select { case <-i.ctx.Done(): return case <-ticker.C: } } }() log.Println("[SchemaIndexer] Started - scanning the store") return nil } // Stop terminates the background rebuilds. func (i *Indexer) Stop() error { i.runMu.Lock() if !i.running { i.runMu.Unlock() return nil } i.running = false i.runMu.Unlock() i.cancel() i.wg.Wait() log.Println("[SchemaIndexer] Stopped") return nil } // Rebuild recomputes the schema from every record key and publishes it. One // rebuild runs at a time; a concurrent caller waits and gets the fresh result. func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) { i.rebuildMu.Lock() defer i.rebuildMu.Unlock() started := time.Now() fields := make(map[string]*fieldState) err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error { for _, row := range rows { fieldPath, keyHint, ok := parseFieldPath(row.K) if !ok { continue } fs := fields[fieldPath] if fs == nil { fs = newFieldState() fields[fieldPath] = fs } if keyHint != "" { fs.languageKeyed = true } prim, hint := ClassifyValue(row.V, true) incContribution(fs, labelPair{prim: prim, hint: hint}) } return ctx.Err() }) if err != nil { return nil, err } snap := &SchemaResponse{ Fields: make(map[string]*FieldSchema, len(fields)), GeneratedAt: time.Now().UTC(), Source: "scan", } for fp, fs := range fields { snap.Fields[fp] = computeFieldSchema(fs) } i.mu.Lock() i.snap = snap i.mu.Unlock() log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond)) return snap, nil } type labelPair struct { prim Primitive hint Hint } // parseFieldPath turns "file:/" into the logical field // path used as the map key. rootID is opaque to the indexer — currently a // UUIDv7, historically a midhash256:abc token; both shapes parse identically // because the split is on the first '/' after "file:". // // When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as // a language code: the field name collapses to the prefix and key_hint is set. func parseFieldPath(key string) (string, KeyHint, bool) { if !strings.HasPrefix(key, "file:") { return "", "", false } rest := strings.TrimPrefix(key, "file:") slash := strings.Index(rest, "/") if slash < 0 { return "", "", false } property := rest[slash+1:] if property == "" { return "", "", false } if last := strings.LastIndex(property, "/"); last >= 0 { tail := property[last+1:] if langCodeRegex.MatchString(tail) { return property[:last], KeyHintLanguageCode, true } } return property, "", true } func incContribution(fs *fieldState, lp labelPair) { label := breakdownLabel(lp.prim, lp.hint) fs.counts[label]++ if lp.prim == PrimUndefined { fs.undefinedCount++ return } fs.typedCount++ fs.primitives[lp.prim]++ if lp.hint != "" { fs.hintCounts[lp.hint]++ } } // Snapshot returns the latest computed schema. Before the first scan finishes // it is empty with Source "pending". func (i *Indexer) Snapshot() *SchemaResponse { i.mu.RLock() defer i.mu.RUnlock() if i.snap == nil { return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"} } return i.snap } func computeFieldSchema(fs *fieldState) *FieldSchema { out := &FieldSchema{ Breakdown: make(map[string]int, len(fs.counts)), } for k, v := range fs.counts { out.Breakdown[k] = v } nPrim := len(fs.primitives) hasUndef := fs.undefinedCount > 0 switch { case nPrim == 0 && hasUndef: out.Type = PrimUndefined case nPrim == 1 && !hasUndef: for p := range fs.primitives { out.Type = p } default: out.Type = PrimMixed } // Strict hint promotion: exactly one hint observed, and it covers every // typed (non-undefined) value. Hints are never set on mixed fields. if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 { for h, c := range fs.hintCounts { if c == fs.typedCount { out.Hint = h } } } if fs.languageKeyed { out.KeyHint = KeyHintLanguageCode } return out }