130 lines
3.9 KiB
Go
130 lines
3.9 KiB
Go
package events
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/metazla/meta-core/internal/storage"
|
|
)
|
|
|
|
const (
|
|
// MetaEventsStream is the stream name for metadata events.
|
|
MetaEventsStream = storage.MetaEventsStream
|
|
|
|
// MetaEventsMaxLen is the approximate cap on stream length. Bounded
|
|
// retention is a contract requirement for the SSE wire — see
|
|
// docs/api-mediated-access.md "Backing-store retention and lifecycle".
|
|
MetaEventsMaxLen = storage.MetaEventsMaxLen
|
|
)
|
|
|
|
// republishBatch is how many events one republish transaction appends —
|
|
// large enough to amortise the commit, small enough that the writer is never
|
|
// held for long.
|
|
const republishBatch = 5_000
|
|
|
|
// MetaPublisher owns the meta:events stream's bulk republish.
|
|
//
|
|
// Live change events no longer pass through here: under Redis this subscribed
|
|
// to keyspace notifications and copied each into the stream, a hop that could
|
|
// drop events whenever the subscriber lagged. The storage engine now appends
|
|
// the event in the same transaction as the change (internal/kv), so a change
|
|
// and its event commit together.
|
|
type MetaPublisher struct {
|
|
client *storage.Client
|
|
running bool
|
|
mu sync.RWMutex
|
|
}
|
|
|
|
// NewMetaPublisher creates a new MetaPublisher.
|
|
func NewMetaPublisher(client *storage.Client) *MetaPublisher {
|
|
return &MetaPublisher{client: client}
|
|
}
|
|
|
|
// Start marks the publisher ready. The stream is NOT truncated on startup —
|
|
// the SSE wire contract requires Last-Event-ID resumption to survive a
|
|
// meta-core restart. Growth is bounded by MetaEventsMaxLen instead.
|
|
func (p *MetaPublisher) Start() error {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.running = true
|
|
log.Println("[MetaPublisher] Started")
|
|
return nil
|
|
}
|
|
|
|
// Stop stops the publisher.
|
|
func (p *MetaPublisher) Stop() error {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
p.running = false
|
|
log.Println("[MetaPublisher] Stopped")
|
|
return nil
|
|
}
|
|
|
|
// IsRunning returns true if the publisher is running.
|
|
func (p *MetaPublisher) IsRunning() bool {
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.running
|
|
}
|
|
|
|
// RepublishFunc is a function type for getting metadata that can be republished
|
|
type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error)
|
|
|
|
// RepublishAllMetadata publishes a `set` event for every property of every
|
|
// record. Called on startup and via the API.
|
|
//
|
|
// It exists for consumers that rebuild their state by replaying meta:events
|
|
// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather
|
|
// than backfilling over HTTP. Appended in bounded batches, each its own
|
|
// transaction, so a multi-million-event republish never holds the writer.
|
|
func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) {
|
|
if !p.IsRunning() {
|
|
return 0, fmt.Errorf("publisher not running")
|
|
}
|
|
|
|
hashIDs, getMetadata, err := republishFunc()
|
|
if err != nil {
|
|
return 0, fmt.Errorf("failed to get hash IDs: %w", err)
|
|
}
|
|
|
|
log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs))
|
|
|
|
published := 0
|
|
batch := make([]map[string]string, 0, republishBatch)
|
|
flush := func() error {
|
|
if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil {
|
|
return err
|
|
}
|
|
published += len(batch)
|
|
batch = batch[:0]
|
|
return nil
|
|
}
|
|
for _, hashID := range hashIDs {
|
|
metadata, err := getMetadata(hashID)
|
|
if err != nil || metadata == nil {
|
|
continue
|
|
}
|
|
ts := fmt.Sprintf("%d", time.Now().UnixMilli())
|
|
for property := range metadata {
|
|
batch = append(batch, map[string]string{
|
|
"type": "set",
|
|
"key": fmt.Sprintf("file:%s/%s", hashID, property),
|
|
"ts": ts,
|
|
})
|
|
if len(batch) >= republishBatch {
|
|
if err := flush(); err != nil {
|
|
return published, fmt.Errorf("republish: %w", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err := flush(); err != nil {
|
|
return published, fmt.Errorf("republish: %w", err)
|
|
}
|
|
|
|
log.Printf("[MetaPublisher] Republished %d property events", published)
|
|
return published, nil
|
|
}
|