replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/api/sse_events.go

@@ -3,7 +3,6 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -11,15 +10,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
-// SSE event stream endpoints for the file:events and meta:events Redis Streams.
+// SSE event stream endpoints for the file:events and meta:events streams.
 //
 // Wire contract (see docs/api-mediated-access.md):
-//   - One SSE event per Redis Stream entry.
-//   - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
-//     MUST treat it as a black-box string they echo back via Last-Event-ID.
+//   - One SSE event per stream entry.
+//   - `id:` is the opaque stream entry ID (`<ms>-<seq>`, the Redis Streams
+//     form, kept across the move to SQLite so cursors consumers persisted to
+//     disk still compare correctly); consumers MUST treat it as a black-box
+//     string they echo back via Last-Event-ID.
 //   - `event:` is the `type` field from the underlying entry.
 //   - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
 //   - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
@@ -30,30 +31,30 @@ import (
 const (
 	sseHeartbeatInterval = 30 * time.Second
 
-	// sseReadBlock controls how long each XREAD blocks before returning.
+	// sseReadBlock controls how long each read waits for new entries.
 	// Much shorter than sseHeartbeatInterval so the handler stays responsive
 	// to client disconnects and the heartbeat check runs frequently enough
 	// that the keep-alive fires within a few seconds of its nominal cadence.
 	sseReadBlock = 2 * time.Second
 
-	// sseReadCount caps how many entries one XREAD returns. Small enough
+	// sseReadCount caps how many entries one read returns. Small enough
 	// to keep per-flush memory low, large enough to drain bursts cheaply.
 	sseReadCount = 100
 )
 
-// handleEventsFiles streams the file:events Redis Stream as SSE.
+// handleEventsFiles streams the file:events stream as SSE.
 func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "file:events")
 }
 
-// handleEventsMeta streams the meta:events Redis Stream as SSE.
+// handleEventsMeta streams the meta:events stream as SSE.
 func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "meta:events")
 }
 
-// streamSSE is the shared SSE pump shared by both endpoints. One handler per
-// stream — there is no fan-out; each subscriber holds its own XREAD loop.
-// That's fine at our scale (≤20 subscribers; one per consumer service).
+// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads
+// the stream itself and sleeps on the store's commit broadcast between reads —
+// fine at our scale (≤20 subscribers; one per consumer service).
 func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
 	if !s.storage.IsConnected() {
 		w.Header().Set("Retry-After", "5")
@@ -89,14 +90,14 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	w.WriteHeader(http.StatusOK)
 	flusher.Flush()
 
-	client := s.storage.GetRedisClient()
-	if client == nil {
+	store := s.storage.Store()
+	if store == nil {
 		// Storage was just disconnected — close cleanly.
 		return
 	}
 
 	ctx := r.Context()
-	cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
+	cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher)
 
 	lastWrite := time.Now()
 	for {
@@ -104,37 +105,34 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 			return
 		}
 
-		// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
-		// context's deadline isn't propagated into XREAD's BLOCK arg —
-		// that's a poll-style timeout on Redis itself — so we keep BLOCK
-		// short.
-		readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
-		result, err := client.XRead(readCtx, &redis.XReadArgs{
-			Streams: []string{stream, cursor},
-			Block:   sseReadBlock,
-			Count:   sseReadCount,
-		}).Result()
-		cancel()
-
+		// Take the wake-up channel BEFORE reading, so an append landing
+		// between the read and the wait still wakes us.
+		wake := store.Wait(stream)
+		entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount)
 		if err != nil {
-			// redis.Nil = BLOCK timeout with no entries. Anything else
-			// while the client is still connected is a real failure.
-			if errors.Is(err, redis.Nil) || isContextTimeout(err) {
-				// fall through to heartbeat check
-			} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
-				return
-			} else {
-				log.Printf("[SSE %s] XREAD error: %v", stream, err)
+			if ctx.Err() != nil {
 				return
 			}
+			log.Printf("[SSE %s] read error: %v", stream, err)
+			return
 		}
 
-		for _, str := range result {
-			for _, msg := range str.Messages {
-				writeSSEEntry(w, flusher, msg)
-				cursor = msg.ID
-				lastWrite = time.Now()
+		for _, e := range entries {
+			writeSSEEntry(w, flusher, e)
+			cursor = e.ID
+			lastWrite = time.Now()
+		}
+
+		if len(entries) == 0 {
+			timer := time.NewTimer(sseReadBlock)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return
+			case <-wake:
+			case <-timer.C:
 			}
+			timer.Stop()
 		}
 
 		// Heartbeat if quiet for >= the interval. Comment line; SSE
@@ -150,38 +148,44 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 }
 
 // resolveStartCursor decides where to start reading.
-//   - No Last-Event-ID → start at "$" (only new entries).
+//   - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
+//     entries appended from now on). Redis' "$" was re-resolved on every
+//     XREAD, which dropped anything appended between two reads.
 //   - Last-Event-ID provided AND still in stream → resume from it.
 //   - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
 //     and resume from the oldest available entry's predecessor "0", so the
-//     next XREAD picks up everything still in the stream.
+//     next read picks up everything still in the stream.
 //
-// The "still in stream" check is cheap: one XRANGE for the first entry.
-func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
-	if lastID == "" {
-		return "$"
-	}
-
-	// One-entry XRANGE to find the oldest ID currently in the stream.
+// The "still in stream" check is cheap: one read of the first entry.
+func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
 	xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
-	entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
+
+	if lastID == "" {
+		head, err := store.LastID(xCtx, stream)
+		if err != nil {
+			log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err)
+			return "0"
+		}
+		return head
+	}
+
+	oldestEntry, ok, err := store.Oldest(xCtx, stream)
 	if err != nil {
 		// Treat any error as "stream not available right now" and resume
-		// from the cursor — XREAD will either find it or block until new
-		// entries arrive.
-		log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
+		// from the cursor — the read either finds it or waits for new entries.
+		log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err)
 		return lastID
 	}
 
-	if len(entries) == 0 {
+	if !ok {
 		// Stream is empty. Cursor may be from a previous incarnation; safest
-		// is to resume from the cursor — XREAD against an empty stream
-		// returns nothing and blocks until new entries arrive.
+		// is to resume from the cursor — reading an empty stream returns
+		// nothing and waits until new entries arrive.
 		return lastID
 	}
 
-	oldest := entries[0].ID
+	oldest := oldestEntry.ID
 	if compareStreamIDs(lastID, oldest) >= 0 {
 		// Cursor is at or past the oldest — fine to resume.
 		return lastID
@@ -197,39 +201,35 @@ func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastI
 	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
 	flusher.Flush()
 
-	// Return "0" so the next XREAD picks up entries starting from the
+	// Return "0" so the next read picks up entries starting from the
 	// stream's true minimum — including `oldest` itself.
 	return "0"
 }
 
-// writeSSEEntry serialises one Redis Stream entry as an SSE event.
+// writeSSEEntry serialises one stream entry as an SSE event.
 //
 // `event:` comes from the entry's `type` field (no prefix munging). `data:`
-// is a JSON object containing all OTHER fields. Field values from the Redis
-// stream are interface{} (typically strings); we encode whatever the stream
-// holds.
-func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
+// is a JSON object containing all OTHER fields.
+func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) {
 	eventType := "message"
-	data := make(map[string]interface{}, len(msg.Values))
-	for k, v := range msg.Values {
+	data := make(map[string]string, len(e.Values))
+	for k, v := range e.Values {
 		if k == "type" {
-			if s, ok := v.(string); ok {
-				eventType = s
-				continue
-			}
+			eventType = v
+			continue
 		}
 		data[k] = v
 	}
 
 	payload, err := json.Marshal(data)
 	if err != nil {
-		log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
+		log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err)
 		return
 	}
 
 	// Spec: each event ends with a blank line. id/event/data lines are
 	// terminated with single \n.
-	fmt.Fprintf(w, "id: %s\n", msg.ID)
+	fmt.Fprintf(w, "id: %s\n", e.ID)
 	fmt.Fprintf(w, "event: %s\n", eventType)
 	fmt.Fprintf(w, "data: %s\n\n", string(payload))
 	flusher.Flush()
@@ -263,18 +263,3 @@ func parseStreamID(id string) (int64, int64) {
 	}
 	return ms, seq
 }
-
-// isContextTimeout returns true for the context deadline error wrapped by
-// go-redis when XREAD's read context fires. We treat it the same as
-// redis.Nil — i.e. an empty poll cycle, not a fatal error.
-func isContextTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	if errors.Is(err, context.DeadlineExceeded) {
-		return true
-	}
-	// go-redis sometimes wraps this in its own error type. String match is
-	// a safety net.
-	return strings.Contains(err.Error(), "context deadline exceeded")
-}