package events import ( "fmt" "log" "sync" "time" "github.com/metazla/meta-core/internal/storage" ) const ( // MetaEventsStream is the stream name for metadata events. MetaEventsStream = storage.MetaEventsStream // MetaEventsMaxLen is the approximate cap on stream length. Bounded // retention is a contract requirement for the SSE wire — see // docs/api-mediated-access.md "Backing-store retention and lifecycle". MetaEventsMaxLen = storage.MetaEventsMaxLen ) // republishBatch is how many events one republish transaction appends — // large enough to amortise the commit, small enough that the writer is never // held for long. const republishBatch = 5_000 // MetaPublisher owns the meta:events stream's bulk republish. // // Live change events no longer pass through here: under Redis this subscribed // to keyspace notifications and copied each into the stream, a hop that could // drop events whenever the subscriber lagged. The storage engine now appends // the event in the same transaction as the change (internal/kv), so a change // and its event commit together. type MetaPublisher struct { client *storage.Client running bool mu sync.RWMutex } // NewMetaPublisher creates a new MetaPublisher. func NewMetaPublisher(client *storage.Client) *MetaPublisher { return &MetaPublisher{client: client} } // Start marks the publisher ready. The stream is NOT truncated on startup — // the SSE wire contract requires Last-Event-ID resumption to survive a // meta-core restart. Growth is bounded by MetaEventsMaxLen instead. func (p *MetaPublisher) Start() error { p.mu.Lock() defer p.mu.Unlock() p.running = true log.Println("[MetaPublisher] Started") return nil } // Stop stops the publisher. func (p *MetaPublisher) Stop() error { p.mu.Lock() defer p.mu.Unlock() p.running = false log.Println("[MetaPublisher] Stopped") return nil } // IsRunning returns true if the publisher is running. func (p *MetaPublisher) IsRunning() bool { p.mu.RLock() defer p.mu.RUnlock() return p.running } // RepublishFunc is a function type for getting metadata that can be republished type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error) // RepublishAllMetadata publishes a `set` event for every property of every // record. Called on startup and via the API. // // It exists for consumers that rebuild their state by replaying meta:events // from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather // than backfilling over HTTP. Appended in bounded batches, each its own // transaction, so a multi-million-event republish never holds the writer. func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) { if !p.IsRunning() { return 0, fmt.Errorf("publisher not running") } hashIDs, getMetadata, err := republishFunc() if err != nil { return 0, fmt.Errorf("failed to get hash IDs: %w", err) } log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs)) published := 0 batch := make([]map[string]string, 0, republishBatch) flush := func() error { if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil { return err } published += len(batch) batch = batch[:0] return nil } for _, hashID := range hashIDs { metadata, err := getMetadata(hashID) if err != nil || metadata == nil { continue } ts := fmt.Sprintf("%d", time.Now().UnixMilli()) for property := range metadata { batch = append(batch, map[string]string{ "type": "set", "key": fmt.Sprintf("file:%s/%s", hashID, property), "ts": ts, }) if len(batch) >= republishBatch { if err := flush(); err != nil { return published, fmt.Errorf("republish: %w", err) } } } } if err := flush(); err != nil { return published, fmt.Errorf("republish: %w", err) } log.Printf("[MetaPublisher] Republished %d property events", published) return published, nil }