replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/events/meta_publisher.go

@@ -1,199 +1,89 @@
 package events
 
 import (
-	"context"
 	"fmt"
 	"log"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/redis/go-redis/v9"
-
 	"github.com/metazla/meta-core/internal/storage"
 )
 
 const (
-	// MetaEventsStream is the Redis stream name for metadata events
-	MetaEventsStream = "meta:events"
+	// MetaEventsStream is the stream name for metadata events.
+	MetaEventsStream = storage.MetaEventsStream
 
 	// MetaEventsMaxLen is the approximate cap on stream length. Bounded
 	// retention is a contract requirement for the SSE wire — see
 	// docs/api-mediated-access.md "Backing-store retention and lifecycle".
-	// Cheap (~) trim keeps growth bounded without serialising writes.
-	MetaEventsMaxLen = 100_000
+	MetaEventsMaxLen = storage.MetaEventsMaxLen
 )
 
-// MetaPublisher subscribes to Redis keyspace notifications
-// and publishes metadata change events to the meta:events stream
+// republishBatch is how many events one republish transaction appends —
+// large enough to amortise the commit, small enough that the writer is never
+// held for long.
+const republishBatch = 5_000
+
+// MetaPublisher owns the meta:events stream's bulk republish.
+//
+// Live change events no longer pass through here: under Redis this subscribed
+// to keyspace notifications and copied each into the stream, a hop that could
+// drop events whenever the subscriber lagged. The storage engine now appends
+// the event in the same transaction as the change (internal/kv), so a change
+// and its event commit together.
 type MetaPublisher struct {
-	client  *redis.Client
-	ctx     context.Context
-	cancel  context.CancelFunc
-	wg      sync.WaitGroup
+	client  *storage.Client
 	running bool
 	mu      sync.RWMutex
 }
 
-// NewMetaPublisher creates a new MetaPublisher
-func NewMetaPublisher(client *redis.Client) *MetaPublisher {
-	ctx, cancel := context.WithCancel(context.Background())
-	return &MetaPublisher{
-		client: client,
-		ctx:    ctx,
-		cancel: cancel,
-	}
+// NewMetaPublisher creates a new MetaPublisher.
+func NewMetaPublisher(client *storage.Client) *MetaPublisher {
+	return &MetaPublisher{client: client}
 }
 
-// Start begins subscribing to keyspace notifications.
-//
-// The stream is NOT truncated on startup — the SSE wire contract requires
-// Last-Event-ID resumption to survive a meta-core restart, which means
-// entries must persist across restart. Growth is bounded instead by
-// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
+// Start marks the publisher ready. The stream is NOT truncated on startup —
+// the SSE wire contract requires Last-Event-ID resumption to survive a
+// meta-core restart. Growth is bounded by MetaEventsMaxLen instead.
 func (p *MetaPublisher) Start() error {
 	p.mu.Lock()
-	if p.running {
-		p.mu.Unlock()
-		return nil
-	}
+	defer p.mu.Unlock()
 	p.running = true
-	p.mu.Unlock()
-
-	p.wg.Add(1)
-	go p.subscribeLoop()
-
-	log.Println("[MetaPublisher] Started - listening for keyspace notifications")
+	log.Println("[MetaPublisher] Started")
 	return nil
 }
 
-// Stop stops the publisher
+// Stop stops the publisher.
 func (p *MetaPublisher) Stop() error {
 	p.mu.Lock()
-	if !p.running {
-		p.mu.Unlock()
-		return nil
-	}
+	defer p.mu.Unlock()
 	p.running = false
-	p.mu.Unlock()
-
-	p.cancel()
-	p.wg.Wait()
-
 	log.Println("[MetaPublisher] Stopped")
 	return nil
 }
 
-// IsRunning returns true if the publisher is running
+// IsRunning returns true if the publisher is running.
 func (p *MetaPublisher) IsRunning() bool {
 	p.mu.RLock()
 	defer p.mu.RUnlock()
 	return p.running
 }
 
-// subscribeLoop subscribes to keyspace notifications and publishes events
-func (p *MetaPublisher) subscribeLoop() {
-	defer p.wg.Done()
-
-	// Subscribe to all file:* keyspace events
-	// Pattern: __keyspace@0__:file:*
-	pattern := "__keyspace@0__:file:*"
-	pubsub := p.client.PSubscribe(p.ctx, pattern)
-	defer pubsub.Close()
-
-	log.Printf("[MetaPublisher] Subscribed to pattern: %s", pattern)
-
-	ch := pubsub.Channel()
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		case msg := <-ch:
-			if msg == nil {
-				continue
-			}
-			// msg.Channel: __keyspace@0__:file:midhash256:abc123/tmdb
-			// msg.Payload: set, del, expire, etc.
-			p.publishEvent(msg.Channel, msg.Payload)
-		}
-	}
-}
-
-// publishEvent publishes a keyspace event to the meta:events stream
-func (p *MetaPublisher) publishEvent(channel, operation string) {
-	// Extract key from channel: __keyspace@0__:file:xxx/field -> file:xxx/field
-	key := strings.TrimPrefix(channel, "__keyspace@0__:")
-
-	// Skip index key and non-file keys
-	if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
-		return
-	}
-	// Skip the per-record field index. It is bookkeeping (see
-	// storage.FieldsField), not a metadata field — publishing it would make
-	// every downstream consumer that parses `file:<id>/<field>` off this
-	// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
-	// field change on a key they can never read.
-	if strings.HasSuffix(key, "/"+storage.FieldsField) {
-		return
-	}
-
-	// Build event fields
-	fields := map[string]interface{}{
-		"type": operation, // "set", "del", "expire"
-		"key":  key,       // "file:midhash256:abc123/tmdb"
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	// Approximate MAXLEN trim keeps the stream bounded across restarts
-	// (see Start). Cap is intentionally generous; most consumers reconnect
-	// well within retention.
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing event: %v", err)
-	}
-}
-
-// publishEventDirect publishes an event directly (not from keyspace notification)
-func (p *MetaPublisher) publishEventDirect(operation, key string) {
-	fields := map[string]interface{}{
-		"type": operation,
-		"key":  key,
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing direct event: %v", err)
-	}
-}
-
 // RepublishFunc is a function type for getting metadata that can be republished
 type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error)
 
-// RepublishAllMetadata iterates over all existing file metadata and publishes events
-// This is called on startup and can be triggered via API
+// RepublishAllMetadata publishes a `set` event for every property of every
+// record. Called on startup and via the API.
+//
+// It exists for consumers that rebuild their state by replaying meta:events
+// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather
+// than backfilling over HTTP. Appended in bounded batches, each its own
+// transaction, so a multi-million-event republish never holds the writer.
 func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) {
-	p.mu.RLock()
-	running := p.running
-	p.mu.RUnlock()
-
-	if !running {
+	if !p.IsRunning() {
 		return 0, fmt.Errorf("publisher not running")
 	}
 
-	// Get all hash IDs and metadata getter
 	hashIDs, getMetadata, err := republishFunc()
 	if err != nil {
 		return 0, fmt.Errorf("failed to get hash IDs: %w", err)
@@ -201,22 +91,39 @@ func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int,
 
 	log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs))
 
-	publishedCount := 0
+	published := 0
+	batch := make([]map[string]string, 0, republishBatch)
+	flush := func() error {
+		if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil {
+			return err
+		}
+		published += len(batch)
+		batch = batch[:0]
+		return nil
+	}
 	for _, hashID := range hashIDs {
-		// Get all properties for this file
 		metadata, err := getMetadata(hashID)
 		if err != nil || metadata == nil {
 			continue
 		}
-
-		// Publish a set event for each property
+		ts := fmt.Sprintf("%d", time.Now().UnixMilli())
 		for property := range metadata {
-			key := fmt.Sprintf("file:%s/%s", hashID, property)
-			p.publishEventDirect("set", key)
-			publishedCount++
+			batch = append(batch, map[string]string{
+				"type": "set",
+				"key":  fmt.Sprintf("file:%s/%s", hashID, property),
+				"ts":   ts,
+			})
+			if len(batch) >= republishBatch {
+				if err := flush(); err != nil {
+					return published, fmt.Errorf("republish: %w", err)
+				}
+			}
 		}
 	}
+	if err := flush(); err != nil {
+		return published, fmt.Errorf("republish: %w", err)
+	}
 
-	log.Printf("[MetaPublisher] Republished %d property events", publishedCount)
-	return publishedCount, nil
+	log.Printf("[MetaPublisher] Republished %d property events", published)
+	return published, nil
 }