v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/api/sse_events.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/metazla/meta-core/internal/kv"
+	"github.com/metazla/meta-core/internal/storage"
 )
 
 // SSE event stream endpoints for the file:events and meta:events streams.
@@ -97,7 +98,22 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	}
 
 	ctx := r.Context()
-	cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher)
+	lastID := r.Header.Get("Last-Event-ID")
+	var cursor string
+	if stream == storage.MetaEventsStream && (lastID == "0" || lastID == "0-0") {
+		// "From the beginning" on meta:events means "the whole current
+		// state": stream it from the records themselves, then the live tail.
+		head, err := streamSnapshot(ctx, store, w, flusher)
+		if err != nil {
+			if ctx.Err() == nil {
+				log.Printf("[SSE %s] snapshot failed: %v", stream, err)
+			}
+			return
+		}
+		cursor = head
+	} else {
+		cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher)
+	}
 
 	lastWrite := time.Now()
 	for {
@@ -147,6 +163,54 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	}
 }
 
+// streamSnapshot sends one `set` event per existing record key, then returns
+// the stream position the live tail continues from.
+//
+// This is how a consumer that rebuilds its state from events (meta-fuse sends
+// Last-Event-ID 0-0 on every start) gets ALL of it. The stream itself only
+// retains ~100k entries, so replaying it never could; meta-core used to
+// republish every field into it on boot instead, which took minutes and still
+// lost everything past the retention window.
+//
+// The head is captured BEFORE the snapshot is read, so a change racing the
+// snapshot appears in the tail too — re-applying a `set` is idempotent. Only
+// the final snapshot event carries an `id:`, so a consumer that disconnects
+// mid-snapshot still holds its old cursor and gets a fresh snapshot.
+func streamSnapshot(ctx context.Context, store *kv.Store, w http.ResponseWriter, flusher http.Flusher) (string, error) {
+	head, err := store.LastID(ctx, storage.MetaEventsStream)
+	if err != nil {
+		return "", err
+	}
+	ts := strconv.FormatInt(time.Now().UnixMilli(), 10)
+	pending := ""
+	err = store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
+		for _, row := range rows {
+			if pending != "" {
+				writeSnapshotEvent(w, pending, ts, "")
+			}
+			pending = row.K
+		}
+		flusher.Flush()
+		return ctx.Err()
+	})
+	if err != nil {
+		return "", err
+	}
+	if pending != "" {
+		writeSnapshotEvent(w, pending, ts, head)
+		flusher.Flush()
+	}
+	return head, nil
+}
+
+func writeSnapshotEvent(w http.ResponseWriter, key, ts, id string) {
+	payload, _ := json.Marshal(map[string]string{"key": key, "ts": ts})
+	if id != "" {
+		fmt.Fprintf(w, "id: %s\n", id)
+	}
+	fmt.Fprintf(w, "event: set\ndata: %s\n\n", payload)
+}
+
 // resolveStartCursor decides where to start reading.
 //   - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
 //     entries appended from now on). Redis' "$" was re-resolved on every