improve editor and add type detection
This commit is contained in:
Pierre
@@ -0,0 +1,103 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// cidRegex matches the CID shapes produced by @metazla/meta-hash:
|
||||
// - CIDv1 multibase-base32: every algorithm (crc32, md5, sha1, sha2-256,
|
||||
// sha3-256, sha3-384, midhash256, btih-v2, …) is serialized via
|
||||
// CID.createV1(code, digest).toString(), which always yields a lowercase
|
||||
// base32 string starting with "ba" (the "b" is the multibase tag, the
|
||||
// "a" is the CIDv1 version byte under base32-encoding-of-0x01).
|
||||
// - CIDv0 base58btc: "Qm" + 44 chars, kept for compatibility with other
|
||||
// services that may still emit v0.
|
||||
//
|
||||
// Minimum CIDv1 length 16 chars matches the shortest real output we observed
|
||||
// (crc32: "bafk3eaqerjhiwhy").
|
||||
var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
|
||||
|
||||
// ClassifyValue returns the (primitive, hint) for a Redis value.
|
||||
// exists=false (key missing) and JSON null literal both map to PrimUndefined.
|
||||
// Empty string is a valid string.
|
||||
func ClassifyValue(value string, exists bool) (Primitive, Hint) {
|
||||
if !exists {
|
||||
return PrimUndefined, ""
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "null" {
|
||||
return PrimUndefined, ""
|
||||
}
|
||||
if trimmed == "true" || trimmed == "false" {
|
||||
return PrimBool, ""
|
||||
}
|
||||
if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
|
||||
if isTimestampInt(trimmed) {
|
||||
return PrimInt, HintTimestamp
|
||||
}
|
||||
return PrimInt, ""
|
||||
}
|
||||
if _, err := strconv.ParseFloat(trimmed, 64); err == nil {
|
||||
return PrimFloat, ""
|
||||
}
|
||||
if len(trimmed) > 0 {
|
||||
switch trimmed[0] {
|
||||
case '{':
|
||||
var v map[string]any
|
||||
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||
return PrimJSONObj, ""
|
||||
}
|
||||
case '[':
|
||||
var v []any
|
||||
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||
return PrimJSONArr, ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if cidRegex.MatchString(value) {
|
||||
return PrimString, HintCID
|
||||
}
|
||||
if isTimestampString(value) {
|
||||
return PrimString, HintTimestamp
|
||||
}
|
||||
return PrimString, ""
|
||||
}
|
||||
|
||||
// isTimestampInt accepts unix-seconds in [2001-09, 2100] or unix-millis in the
|
||||
// same range. Anything else is treated as a plain int.
|
||||
func isTimestampInt(s string) bool {
|
||||
n, err := strconv.ParseInt(s, 10, 64)
|
||||
if err != nil || n <= 0 {
|
||||
return false
|
||||
}
|
||||
if n >= 1_000_000_000 && n <= 4_102_444_800 {
|
||||
return true
|
||||
}
|
||||
if n >= 1_000_000_000_000 && n <= 4_102_444_800_000 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isTimestampString(s string) bool {
|
||||
if _, err := time.Parse(time.RFC3339, s); err == nil {
|
||||
return true
|
||||
}
|
||||
if _, err := time.Parse("2006-01-02", s); err == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// breakdownLabel returns the most specific label for a classified value.
|
||||
// Hints take precedence over primitives.
|
||||
func breakdownLabel(prim Primitive, hint Hint) string {
|
||||
if hint != "" {
|
||||
return string(hint)
|
||||
}
|
||||
return string(prim)
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package schema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/events"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
|
||||
|
||||
// fieldState is the per-field accumulator. perKey lets us undo a previous
|
||||
// contribution when the same Redis key is re-SET or DELed.
|
||||
type fieldState struct {
|
||||
counts map[string]int
|
||||
primitives map[Primitive]int
|
||||
hintCounts map[Hint]int
|
||||
typedCount int
|
||||
undefinedCount int
|
||||
languageKeyed bool
|
||||
perKey map[string]labelPair
|
||||
}
|
||||
|
||||
type labelPair struct {
|
||||
prim Primitive
|
||||
hint Hint
|
||||
}
|
||||
|
||||
func newFieldState() *fieldState {
|
||||
return &fieldState{
|
||||
counts: make(map[string]int),
|
||||
primitives: make(map[Primitive]int),
|
||||
hintCounts: make(map[Hint]int),
|
||||
perKey: make(map[string]labelPair),
|
||||
}
|
||||
}
|
||||
|
||||
// Indexer maintains a live field-schema by consuming meta:events.
|
||||
type Indexer struct {
|
||||
client *redis.Client
|
||||
keyPrefix string
|
||||
|
||||
mu sync.RWMutex
|
||||
fields map[string]*fieldState
|
||||
|
||||
processed uint64 // total events processed since start
|
||||
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
runMu sync.Mutex
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
|
||||
// so the indexer can build absolute Redis keys for GET calls.
|
||||
func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Indexer{
|
||||
client: client,
|
||||
keyPrefix: keyPrefix,
|
||||
fields: make(map[string]*fieldState),
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins consuming the meta:events stream from the earliest available ID.
|
||||
func (i *Indexer) Start() error {
|
||||
i.runMu.Lock()
|
||||
if i.running {
|
||||
i.runMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
i.running = true
|
||||
i.runMu.Unlock()
|
||||
|
||||
i.wg.Add(1)
|
||||
go i.consumeLoop()
|
||||
|
||||
log.Println("[SchemaIndexer] Started - consuming meta:events stream")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the consumer goroutine.
|
||||
func (i *Indexer) Stop() error {
|
||||
i.runMu.Lock()
|
||||
if !i.running {
|
||||
i.runMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
i.running = false
|
||||
i.runMu.Unlock()
|
||||
|
||||
i.cancel()
|
||||
i.wg.Wait()
|
||||
log.Println("[SchemaIndexer] Stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
// ProcessedCount returns the total number of stream events processed.
|
||||
// Used by WaitForDrain to detect when a rescan flood has been absorbed.
|
||||
func (i *Indexer) ProcessedCount() uint64 {
|
||||
return atomic.LoadUint64(&i.processed)
|
||||
}
|
||||
|
||||
// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
|
||||
// the timeout elapses. Suitable for use right after RepublishMetadata.
|
||||
func (i *Indexer) WaitForDrain(timeout time.Duration) {
|
||||
deadline := time.Now().Add(timeout)
|
||||
last := i.ProcessedCount()
|
||||
stableSince := time.Now()
|
||||
for time.Now().Before(deadline) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
now := i.ProcessedCount()
|
||||
if now != last {
|
||||
last = now
|
||||
stableSince = time.Now()
|
||||
continue
|
||||
}
|
||||
if time.Since(stableSince) >= 300*time.Millisecond {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Indexer) consumeLoop() {
|
||||
defer i.wg.Done()
|
||||
lastID := "0"
|
||||
for {
|
||||
select {
|
||||
case <-i.ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
|
||||
Streams: []string{events.MetaEventsStream, lastID},
|
||||
Block: 2 * time.Second,
|
||||
Count: 500,
|
||||
}).Result()
|
||||
|
||||
if err != nil {
|
||||
if err == redis.Nil || i.ctx.Err() != nil {
|
||||
continue
|
||||
}
|
||||
log.Printf("[SchemaIndexer] XRead error: %v", err)
|
||||
time.Sleep(1 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, stream := range res {
|
||||
for _, msg := range stream.Messages {
|
||||
lastID = msg.ID
|
||||
i.processEvent(msg.Values)
|
||||
atomic.AddUint64(&i.processed, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Indexer) processEvent(values map[string]any) {
|
||||
op, _ := values["type"].(string)
|
||||
key, _ := values["key"].(string)
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
fieldPath, keyHint, ok := parseFieldPath(key)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
switch op {
|
||||
case "set":
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
|
||||
cancel()
|
||||
exists := err != redis.Nil
|
||||
if err != nil && err != redis.Nil {
|
||||
// Treat transient errors as "value disappeared" — nothing to record.
|
||||
return
|
||||
}
|
||||
prim, hint := ClassifyValue(val, exists)
|
||||
i.upsert(fieldPath, key, keyHint, prim, hint)
|
||||
case "del", "expire":
|
||||
i.remove(fieldPath, key)
|
||||
}
|
||||
}
|
||||
|
||||
// parseFieldPath turns "file:<hashID>/<propertyPath>" into the logical field
|
||||
// path used as the map key. hashID may itself contain ':' (e.g.
|
||||
// "midhash256:abc123"); the split is on the first '/'.
|
||||
//
|
||||
// When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as
|
||||
// a language code: the field name collapses to the prefix and key_hint is set.
|
||||
func parseFieldPath(key string) (string, KeyHint, bool) {
|
||||
if !strings.HasPrefix(key, "file:") {
|
||||
return "", "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(key, "file:")
|
||||
slash := strings.Index(rest, "/")
|
||||
if slash < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
property := rest[slash+1:]
|
||||
if property == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if last := strings.LastIndex(property, "/"); last >= 0 {
|
||||
tail := property[last+1:]
|
||||
if langCodeRegex.MatchString(tail) {
|
||||
return property[:last], KeyHintLanguageCode, true
|
||||
}
|
||||
}
|
||||
return property, "", true
|
||||
}
|
||||
|
||||
func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
fs, ok := i.fields[fieldPath]
|
||||
if !ok {
|
||||
fs = newFieldState()
|
||||
i.fields[fieldPath] = fs
|
||||
}
|
||||
if keyHint != "" {
|
||||
fs.languageKeyed = true
|
||||
}
|
||||
|
||||
if old, ok := fs.perKey[redisKey]; ok {
|
||||
decContribution(fs, old)
|
||||
}
|
||||
|
||||
fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
|
||||
incContribution(fs, labelPair{prim: prim, hint: hint})
|
||||
}
|
||||
|
||||
func (i *Indexer) remove(fieldPath, redisKey string) {
|
||||
i.mu.Lock()
|
||||
defer i.mu.Unlock()
|
||||
|
||||
fs, ok := i.fields[fieldPath]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
old, ok := fs.perKey[redisKey]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(fs.perKey, redisKey)
|
||||
decContribution(fs, old)
|
||||
if len(fs.perKey) == 0 {
|
||||
delete(i.fields, fieldPath)
|
||||
}
|
||||
}
|
||||
|
||||
func incContribution(fs *fieldState, lp labelPair) {
|
||||
label := breakdownLabel(lp.prim, lp.hint)
|
||||
fs.counts[label]++
|
||||
if lp.prim == PrimUndefined {
|
||||
fs.undefinedCount++
|
||||
return
|
||||
}
|
||||
fs.typedCount++
|
||||
fs.primitives[lp.prim]++
|
||||
if lp.hint != "" {
|
||||
fs.hintCounts[lp.hint]++
|
||||
}
|
||||
}
|
||||
|
||||
func decContribution(fs *fieldState, lp labelPair) {
|
||||
label := breakdownLabel(lp.prim, lp.hint)
|
||||
fs.counts[label]--
|
||||
if fs.counts[label] <= 0 {
|
||||
delete(fs.counts, label)
|
||||
}
|
||||
if lp.prim == PrimUndefined {
|
||||
fs.undefinedCount--
|
||||
return
|
||||
}
|
||||
fs.typedCount--
|
||||
fs.primitives[lp.prim]--
|
||||
if fs.primitives[lp.prim] <= 0 {
|
||||
delete(fs.primitives, lp.prim)
|
||||
}
|
||||
if lp.hint != "" {
|
||||
fs.hintCounts[lp.hint]--
|
||||
if fs.hintCounts[lp.hint] <= 0 {
|
||||
delete(fs.hintCounts, lp.hint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot returns the current schema. Safe to call concurrently with consumption.
|
||||
func (i *Indexer) Snapshot() *SchemaResponse {
|
||||
i.mu.RLock()
|
||||
defer i.mu.RUnlock()
|
||||
|
||||
out := &SchemaResponse{
|
||||
Fields: make(map[string]*FieldSchema, len(i.fields)),
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
Source: "live",
|
||||
}
|
||||
for fp, fs := range i.fields {
|
||||
out.Fields[fp] = computeFieldSchema(fs)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func computeFieldSchema(fs *fieldState) *FieldSchema {
|
||||
out := &FieldSchema{
|
||||
Breakdown: make(map[string]int, len(fs.counts)),
|
||||
}
|
||||
for k, v := range fs.counts {
|
||||
out.Breakdown[k] = v
|
||||
}
|
||||
|
||||
nPrim := len(fs.primitives)
|
||||
hasUndef := fs.undefinedCount > 0
|
||||
|
||||
switch {
|
||||
case nPrim == 0 && hasUndef:
|
||||
out.Type = PrimUndefined
|
||||
case nPrim == 1 && !hasUndef:
|
||||
for p := range fs.primitives {
|
||||
out.Type = p
|
||||
}
|
||||
default:
|
||||
out.Type = PrimMixed
|
||||
}
|
||||
|
||||
// Strict hint promotion: exactly one hint observed, and it covers every
|
||||
// typed (non-undefined) value. Hints are never set on mixed fields.
|
||||
if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 {
|
||||
for h, c := range fs.hintCounts {
|
||||
if c == fs.typedCount {
|
||||
out.Hint = h
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if fs.languageKeyed {
|
||||
out.KeyHint = KeyHintLanguageCode
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package schema
|
||||
|
||||
import "time"
|
||||
|
||||
// Primitive is the coarse value type used by the schema indexer.
|
||||
// PrimMixed appears only as an aggregate label on a field, never per value.
|
||||
type Primitive string
|
||||
|
||||
const (
|
||||
PrimString Primitive = "string"
|
||||
PrimInt Primitive = "int"
|
||||
PrimFloat Primitive = "float"
|
||||
PrimBool Primitive = "bool"
|
||||
PrimJSONObj Primitive = "json-object"
|
||||
PrimJSONArr Primitive = "json-array"
|
||||
PrimUndefined Primitive = "undefined"
|
||||
PrimMixed Primitive = "mixed"
|
||||
)
|
||||
|
||||
// Hint is a semantic refinement on top of a primitive (e.g. cid, timestamp).
|
||||
// A hint is promoted onto a field only when every non-undefined value matches it.
|
||||
type Hint string
|
||||
|
||||
const (
|
||||
HintCID Hint = "cid"
|
||||
HintTimestamp Hint = "timestamp"
|
||||
)
|
||||
|
||||
// KeyHint describes the shape of the subkey dimension when a field is keyed
|
||||
// by a structured discriminator (e.g. title/eng, title/fre → language-code).
|
||||
type KeyHint string
|
||||
|
||||
const KeyHintLanguageCode KeyHint = "language-code"
|
||||
|
||||
// FieldSchema is the per-field entry returned by GET /api/schema.
|
||||
type FieldSchema struct {
|
||||
Type Primitive `json:"type"`
|
||||
Hint Hint `json:"hint,omitempty"`
|
||||
KeyHint KeyHint `json:"key_hint,omitempty"`
|
||||
Breakdown map[string]int `json:"breakdown"`
|
||||
}
|
||||
|
||||
// SchemaResponse is the top-level payload for GET /api/schema.
|
||||
type SchemaResponse struct {
|
||||
Fields map[string]*FieldSchema `json:"fields"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
Reference in New Issue
Block a user