replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/kv/kv.go

@@ -0,0 +1,713 @@
+// Package kv is meta-core's storage engine: one string→string table in an
+// embedded SQLite file.
+//
+// # Why a key/value table and not a schema
+//
+// meta-core's data model is schema-free on purpose — a record is a set of flat
+// `file:<rid>/<field>` keys, and a new metadata field is just a new key
+// (METADATA_KEYS.md). The engine keeps exactly that model:
+//
+//	CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID
+//
+// is the only table that holds data, and it never needs a migration. What
+// Redis gave us that a bare KV would not — "list a record's fields", "find
+// records by field value", free-text search — comes from the table being
+// *ordered* (a record is a key range) and from two derived structures the
+// storage package maintains on top: an expression index over the field name
+// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv
+// rows and never authoritative.
+//
+// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a
+// 169k-record corpus (Redis plus the API's in-memory copy needed to search
+// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here
+// and the working set is a bounded page cache.
+//
+// # Concurrency
+//
+// SQLite allows one writer at a time, so every write goes through one writer
+// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED:
+// the writer runs every request waiting at that moment inside one SQLite
+// transaction, each under its own SAVEPOINT (a failing request rolls back
+// alone), and commits once. A commit — writing the dirty pages of the table,
+// the field index and the FTS shadow tables to the WAL — was ~40% of the cost
+// of a record write, so sharing it is what makes concurrent ingest scale.
+//
+// Reads use a separate pool and see the last committed state (WAL). Inside
+// Update, read through the *Tx — the reader pool cannot see uncommitted rows —
+// and never call Update from inside an Update callback (the writer would wait
+// for itself).
+//
+// # Events
+//
+// Redis keyspace notifications are replaced by the transaction itself: every
+// Put/Delete of a key the Options.EventKey predicate selects is appended, in
+// the same transaction, to Options.EventStream as {type: set|del, key, ts}.
+// A change and its event therefore commit together or not at all.
+package kv
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// Options configures a Store.
+type Options struct {
+	// EventStream receives a set/del entry for every changed key EventKey
+	// selects. Empty disables change events.
+	EventStream string
+	// EventMaxLen is the approximate retention of EventStream.
+	EventMaxLen int64
+	// EventKey selects the keys whose changes are published.
+	EventKey func(key string) bool
+	// WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection.
+	WriterCacheMB int
+	ReaderCacheMB int
+	// Readers is the size of the read connection pool.
+	Readers int
+	// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
+	// for derived indexes owned by the caller.
+	Schema string
+}
+
+// Store is an open database.
+type Store struct {
+	path string
+	opts Options
+	w    *sql.DB // single connection: the one writer
+	r    *sql.DB // read pool
+
+	streamMu sync.Mutex
+	streams  map[string]*streamState
+
+	// Hot write statements, prepared once on the writer connection: parsing
+	// SQL on every Put was the single largest cost of a record write.
+	stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
+
+	writes    chan *writeReq
+	writeMu   sync.Mutex // held by the writer while a group runs
+	closeOnce sync.Once
+	stopped   chan struct{}
+}
+
+// writeReq is one Update call waiting for the writer.
+type writeReq struct {
+	fn   func(tx *Tx) error
+	done chan error
+}
+
+// maxGroup caps how many requests share one commit, bounding how long the
+// writer holds a transaction open.
+const maxGroup = 128
+
+// Open opens (creating if needed) the database file at path.
+func Open(path string, opts Options) (*Store, error) {
+	if opts.Readers <= 0 {
+		opts.Readers = 8
+	}
+	if opts.WriterCacheMB <= 0 {
+		opts.WriterCacheMB = 64
+	}
+	if opts.ReaderCacheMB <= 0 {
+		opts.ReaderCacheMB = 16
+	}
+	if opts.EventMaxLen <= 0 {
+		opts.EventMaxLen = 100_000
+	}
+
+	w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false))
+	if err != nil {
+		return nil, fmt.Errorf("open writer: %w", err)
+	}
+	w.SetMaxOpenConns(1)
+	w.SetMaxIdleConns(1)
+	w.SetConnMaxLifetime(0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
+		w.Close()
+		return nil, fmt.Errorf("create kv: %w", err)
+	}
+	if opts.Schema != "" {
+		// No deadline: on an existing store, the first open after a new index
+		// is added builds it over every row (minutes at millions of keys).
+		// Cancelling that just rolls it back and restarts it on the next boot.
+		started := time.Now()
+		if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil {
+			w.Close()
+			return nil, fmt.Errorf("apply schema: %w", err)
+		}
+		if d := time.Since(started); d > 5*time.Second {
+			log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second))
+		}
+	}
+
+	r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true))
+	if err != nil {
+		w.Close()
+		return nil, fmt.Errorf("open readers: %w", err)
+	}
+	r.SetMaxOpenConns(opts.Readers)
+	r.SetMaxIdleConns(opts.Readers)
+
+	st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)}
+	// Fresh deadline: the schema step above may legitimately have taken minutes.
+	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	for _, p := range []struct {
+		dst   **sql.Stmt
+		query string
+	}{
+		{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
+		{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
+		{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
+		{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
+		{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
+	} {
+		if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil {
+			r.Close()
+			w.Close()
+			return nil, fmt.Errorf("prepare: %w", err)
+		}
+	}
+	st.writes = make(chan *writeReq)
+	st.stopped = make(chan struct{})
+	go st.writer()
+	return st, nil
+}
+
+// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class
+// Redis' AOF-everysec gave us: a power loss may drop the last transactions,
+// never corrupt the file.
+func dsn(path string, cacheMB int, readOnly bool) string {
+	q := url.Values{}
+	q.Add("_pragma", "busy_timeout(30000)")
+	q.Add("_pragma", "journal_mode(WAL)")
+	q.Add("_pragma", "synchronous(NORMAL)")
+	q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024))
+	q.Add("_pragma", "temp_store(MEMORY)")
+	q.Add("_pragma", "journal_size_limit(268435456)")
+	if readOnly {
+		q.Add("_pragma", "query_only(1)")
+	} else {
+		q.Add("_txlock", "immediate")
+	}
+	return "file:" + path + "?" + q.Encode()
+}
+
+// Close stops the writer and closes both pools, checkpointing the WAL into the
+// main file.
+func (s *Store) Close() error {
+	s.closeOnce.Do(func() { close(s.stopped) })
+	s.writeMu.Lock() // wait for an in-flight group to finish
+	defer s.writeMu.Unlock()
+	_, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`)
+	return errors.Join(s.r.Close(), s.w.Close())
+}
+
+// Ping reports whether the database answers.
+func (s *Store) Ping(ctx context.Context) error {
+	var one int
+	return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one)
+}
+
+// Path is the database file path.
+func (s *Store) Path() string { return s.path }
+
+// DB exposes the read pool for callers that own derived indexes (search). It
+// is read-only (query_only), so it cannot bypass Update.
+func (s *Store) DB() *sql.DB { return s.r }
+
+// FileSize returns the on-disk size of the database plus its WAL.
+func (s *Store) FileSize() int64 {
+	var total int64
+	for _, suffix := range []string{"", "-wal"} {
+		if fi, err := os.Stat(s.path + suffix); err == nil {
+			total += fi.Size()
+		}
+	}
+	return total
+}
+
+// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer
+// slot and waits for readers, so callers run it when idle.
+func (s *Store) Checkpoint(ctx context.Context) error {
+	_, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
+	return err
+}
+
+// KV is one key/value pair.
+type KV struct {
+	K, V string
+}
+
+// Reader is the read surface shared by Store (committed state) and Tx (the
+// transaction's own view). Storage helpers take a Reader so they work both
+// inside and outside a write.
+type Reader interface {
+	Get(ctx context.Context, k string) (string, bool, error)
+	MGet(ctx context.Context, keys []string) (map[string]string, error)
+	Range(ctx context.Context, prefix, after string, limit int) ([]KV, error)
+	Count(ctx context.Context, prefix string) (int64, error)
+}
+
+type queryer interface {
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
+	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
+}
+
+// PrefixEnd returns the smallest string greater than every string with the
+// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range.
+// An empty result means "unbounded".
+func PrefixEnd(prefix string) string {
+	b := []byte(prefix)
+	for i := len(b) - 1; i >= 0; i-- {
+		if b[i] < 0xff {
+			b[i]++
+			return string(b[:i+1])
+		}
+	}
+	return ""
+}
+
+func get(ctx context.Context, q queryer, k string) (string, bool, error) {
+	var v string
+	err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+
+// mgetBatch stays well under SQLite's default 32766 bound-parameter cap.
+const mgetBatch = 500
+
+func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) {
+	out := make(map[string]string, len(keys))
+	for start := 0; start < len(keys); start += mgetBatch {
+		end := min(start+mgetBatch, len(keys))
+		chunk := keys[start:end]
+		args := make([]any, len(chunk))
+		for i, k := range chunk {
+			args[i] = k
+		}
+		rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...)
+		if err != nil {
+			return nil, err
+		}
+		for rows.Next() {
+			var k, v string
+			if err := rows.Scan(&k, &v); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			out[k] = v
+		}
+		if err := rows.Close(); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+// rangeQuery returns keys with prefix, strictly after `after` (when non-empty),
+// in key order, at most limit (<= 0 means all).
+func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) {
+	lo, loOp := prefix, ">="
+	if after != "" && after >= prefix {
+		lo, loOp = after, ">"
+	}
+	query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?`
+	args := []any{lo}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	query += ` ORDER BY k`
+	if limit > 0 {
+		query += ` LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := q.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		out = append(out, kv)
+	}
+	return out, rows.Err()
+}
+
+func count(ctx context.Context, q queryer, prefix string) (int64, error) {
+	query := `SELECT count(*) FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	var n int64
+	err := q.QueryRowContext(ctx, query, args...).Scan(&n)
+	return n, err
+}
+
+// Get returns the committed value of k.
+func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) }
+
+// MGet returns the committed values of the keys that exist.
+func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, s.r, keys)
+}
+
+// Range returns committed keys under prefix after `after`, in key order.
+func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, s.r, prefix, after, limit)
+}
+
+// Count returns how many committed keys sit under prefix.
+func (s *Store) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, s.r, prefix)
+}
+
+// ScanPrefix walks every committed key under prefix in pages of `page`, each
+// page its own short read so a long walk never pins a WAL snapshot (which
+// would stop checkpoints and let the WAL grow without bound).
+func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error {
+	if page <= 0 {
+		page = 5000
+	}
+	after := ""
+	for {
+		batch, err := rangeQuery(ctx, s.r, prefix, after, page)
+		if err != nil {
+			return err
+		}
+		if len(batch) == 0 {
+			return nil
+		}
+		if err := fn(batch); err != nil {
+			return err
+		}
+		if len(batch) < page {
+			return nil
+		}
+		after = batch[len(batch)-1].K
+	}
+}
+
+// CountAll counts every row. A full scan — callers cache it.
+func (s *Store) CountAll(ctx context.Context) (int64, error) {
+	var n int64
+	err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n)
+	return n, err
+}
+
+// ErrClosed is returned by Update on a closed store.
+var ErrClosed = errors.New("kv: store closed")
+
+// Update runs fn atomically: its writes (and the change events they publish)
+// commit together or not at all. fn may share a SQLite transaction with other
+// concurrent Updates (group commit) but is isolated from their failures by a
+// savepoint. Never call Update from inside fn.
+//
+// ctx bounds only the wait for the writer. Once fn starts it runs to
+// completion: interrupting a statement inside a shared transaction would roll
+// back every request in the group.
+func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error {
+	req := &writeReq{fn: fn, done: make(chan error, 1)}
+	select {
+	case s.writes <- req:
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-s.stopped:
+		return ErrClosed
+	}
+	return <-req.done
+}
+
+// writer is the single goroutine that executes writes.
+func (s *Store) writer() {
+	for {
+		var first *writeReq
+		select {
+		case first = <-s.writes:
+		case <-s.stopped:
+			return
+		}
+		group := []*writeReq{first}
+	drain:
+		for len(group) < maxGroup {
+			select {
+			case r := <-s.writes:
+				group = append(group, r)
+			default:
+				break drain
+			}
+		}
+		s.writeMu.Lock()
+		s.runGroup(group)
+		s.writeMu.Unlock()
+	}
+}
+
+// group is the state shared by the requests of one commit.
+type group struct {
+	tx      *sql.Tx
+	pending map[string]*streamPending
+	stmts   map[*sql.Stmt]*sql.Stmt
+}
+
+func (g *group) snapshot() map[string]streamPending {
+	out := make(map[string]streamPending, len(g.pending))
+	for k, p := range g.pending {
+		out[k] = *p
+	}
+	return out
+}
+
+func (g *group) restore(snap map[string]streamPending) {
+	g.pending = make(map[string]*streamPending, len(snap))
+	for k, p := range snap {
+		p := p
+		g.pending[k] = &p
+	}
+}
+
+func (s *Store) runGroup(reqs []*writeReq) {
+	ctx := context.Background()
+	results := make([]error, len(reqs))
+	fail := func(err error) {
+		for i, r := range reqs {
+			if results[i] == nil {
+				results[i] = err
+			}
+			r.done <- results[i]
+		}
+	}
+
+	sqlTx, err := s.w.BeginTx(ctx, nil)
+	if err != nil {
+		fail(fmt.Errorf("begin: %w", err))
+		return
+	}
+	g := &group{tx: sqlTx}
+	for i, r := range reqs {
+		if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("savepoint: %w", err))
+			return
+		}
+		snap := g.snapshot()
+		tx := &Tx{ctx: ctx, s: s, g: g}
+		err := r.fn(tx)
+		if err == nil {
+			err = tx.flushEvents()
+		}
+		if err != nil {
+			results[i] = err
+			if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
+				_ = sqlTx.Rollback()
+				fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err))
+				return
+			}
+			g.restore(snap)
+		}
+		if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("release savepoint: %w", err))
+			return
+		}
+	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
+	if err := sqlTx.Commit(); err != nil {
+		fail(fmt.Errorf("commit: %w", err))
+		return
+	}
+	s.afterCommit(g.pending)
+	for i, r := range reqs {
+		r.done <- results[i]
+	}
+}
+
+// Tx is one Update's view of the write transaction. Its reads see its own
+// writes (and those of earlier requests in the same group).
+type Tx struct {
+	ctx    context.Context
+	s      *Store
+	g      *group
+	events []map[string]string
+}
+
+func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
+
+// stmt binds one of the store's prepared statements to the transaction.
+func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt {
+	if t.g.stmts == nil {
+		t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5)
+	}
+	bound := t.g.stmts[st]
+	if bound == nil {
+		bound = t.g.tx.StmtContext(t.ctx, st)
+		t.g.stmts[st] = bound
+	}
+	return bound
+}
+
+// Context is the context the transaction runs under.
+func (t *Tx) Context() context.Context { return t.ctx }
+
+func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) {
+	var v string
+	err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, t.sqlTx(), keys)
+}
+func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, t.sqlTx(), prefix, after, limit)
+}
+func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, t.sqlTx(), prefix)
+}
+
+// Exec runs a raw statement in the transaction — for the caller's derived
+// tables (FTS), never for kv rows (which would skip events).
+func (t *Tx) Exec(query string, args ...any) (sql.Result, error) {
+	return t.sqlTx().ExecContext(t.ctx, query, args...)
+}
+
+// QueryRow runs a raw single-row query in the transaction.
+func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
+	return t.sqlTx().QueryRowContext(t.ctx, query, args...)
+}
+
+// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
+// keyspace notifications did.
+func (t *Tx) Put(k, v string) error {
+	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
+		return fmt.Errorf("put %q: %w", k, err)
+	}
+	t.event("set", k)
+	return nil
+}
+
+// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
+// Publishes nothing when the key already existed (nothing changed).
+func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
+	res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	n, _ := res.RowsAffected()
+	if n > 0 {
+		t.event("set", k)
+	}
+	return n > 0, nil
+}
+
+// Delete removes k. Reports whether it existed; publishes `del` only then,
+// as Redis did.
+func (t *Tx) Delete(k string) (bool, error) {
+	res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
+	if err != nil {
+		return false, fmt.Errorf("delete %q: %w", k, err)
+	}
+	n, _ := res.RowsAffected()
+	if n > 0 {
+		t.event("del", k)
+	}
+	return n > 0, nil
+}
+
+// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
+// returns how many went. Bulk callers loop over separate Updates with a limit
+// so one transaction never grows to millions of rows.
+func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
+	end := PrefixEnd(prefix)
+	sel := `SELECT k FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end != "" {
+		sel += ` AND k < ?`
+		args = append(args, end)
+	}
+	if limit > 0 {
+		sel += ` ORDER BY k LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
+	if err != nil {
+		return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
+	}
+	var keys []string
+	for rows.Next() {
+		var k string
+		if err := rows.Scan(&k); err != nil {
+			rows.Close()
+			return 0, err
+		}
+		keys = append(keys, k)
+	}
+	if err := rows.Close(); err != nil {
+		return 0, err
+	}
+	for _, k := range keys {
+		t.event("del", k)
+	}
+	return int64(len(keys)), nil
+}
+
+func (t *Tx) event(op, k string) {
+	if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) {
+		return
+	}
+	t.events = append(t.events, map[string]string{
+		"type": op,
+		"key":  k,
+		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
+	})
+}
+
+// flushEvents appends the queued change events to the event stream, inside
+// the transaction, right before commit.
+func (t *Tx) flushEvents() error {
+	if len(t.events) == 0 {
+		return nil
+	}
+	evs := t.events
+	t.events = nil
+	for _, e := range evs {
+		if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/kv/kv_test.go

@@ -0,0 +1,314 @@
+package kv
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func openTest(t *testing.T, path string) *Store {
+	t.Helper()
+	if path == "" {
+		path = filepath.Join(t.TempDir(), "kv.db")
+	}
+	s, err := Open(path, Options{
+		EventStream: "meta:events",
+		EventMaxLen: 100_000,
+		EventKey:    func(k string) bool { return strings.HasPrefix(k, "file:") },
+	})
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func update(t *testing.T, s *Store, fn func(tx *Tx) error) {
+	t.Helper()
+	if err := s.Update(context.Background(), fn); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+}
+
+func events(t *testing.T, s *Store) []Entry {
+	t.Helper()
+	es, err := s.ReadAfter(context.Background(), "meta:events", "0", 0)
+	if err != nil {
+		t.Fatalf("read events: %v", err)
+	}
+	return es
+}
+
+// The replacement for keyspace notifications: a change and its event commit
+// together, with Redis' semantics — `set` even for an unchanged value, `del`
+// only for a key that existed, and nothing for keys outside the event scope.
+func TestChangesPublishEventsWithRedisSemantics(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		_ = tx.Put("file:A/title", "x") // unchanged value still publishes
+		_ = tx.Put("cid:bafk", "A")     // outside the event scope
+		_, _ = tx.Delete("file:A/missing")
+		_, _ = tx.Delete("file:A/title")
+		return nil
+	})
+	got := events(t, s)
+	want := []string{"set file:A/title", "set file:A/title", "del file:A/title"}
+	if len(got) != len(want) {
+		t.Fatalf("events = %+v, want %v", got, want)
+	}
+	for i, e := range got {
+		if e.Values["type"]+" "+e.Values["key"] != want[i] {
+			t.Errorf("event %d = %v, want %q", i, e.Values, want[i])
+		}
+	}
+}
+
+func TestRolledBackChangesPublishNothing(t *testing.T) {
+	s := openTest(t, "")
+	err := s.Update(context.Background(), func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		return errors.New("boom")
+	})
+	if err == nil {
+		t.Fatal("want the error back")
+	}
+	if _, ok, _ := s.Get(context.Background(), "file:A/title"); ok {
+		t.Fatal("rolled-back write is visible")
+	}
+	if es := events(t, s); len(es) != 0 {
+		t.Fatalf("rolled-back write published %+v", es)
+	}
+}
+
+func TestDeletePrefixPublishesOneDelPerKey(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b", "file:AB/c"} {
+			_ = tx.Put(k, "1")
+		}
+		return nil
+	})
+	var n int64
+	update(t, s, func(tx *Tx) error {
+		var err error
+		n, err = tx.DeletePrefix("file:A/", 0)
+		return err
+	})
+	if n != 2 {
+		t.Fatalf("deleted %d, want 2 (file:AB/ is not under file:A/)", n)
+	}
+	es := events(t, s)
+	if dels := es[3:]; len(dels) != 2 || dels[0].Values["type"] != "del" || dels[1].Values["type"] != "del" {
+		t.Fatalf("want two del events, got %+v", es)
+	}
+}
+
+// IDs are Redis-shaped and strictly increasing, within one millisecond and
+// across a reopen — the SSE cursors consumers persist depend on both.
+func TestStreamIDsAreMonotonicAcrossReopen(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "kv.db")
+	s, err := Open(path, Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ids []string
+	if err := s.Update(context.Background(), func(tx *Tx) error {
+		for i := 0; i < 50; i++ {
+			id, err := tx.Append("file:events", 0, map[string]string{"n": "x"})
+			if err != nil {
+				return err
+			}
+			ids = append(ids, id)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	_ = s.Close()
+
+	s2 := openTest(t, path)
+	var after string
+	update(t, s2, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"n": "y"})
+		return err
+	})
+	ids = append(ids, after)
+	for i := 1; i < len(ids); i++ {
+		if !idLess(t, ids[i-1], ids[i]) {
+			t.Fatalf("id %d (%s) does not follow %s", i, ids[i], ids[i-1])
+		}
+	}
+	if !strings.Contains(ids[0], "-") || strings.HasPrefix(ids[0], "0") {
+		t.Fatalf("id %q is not the unpadded <ms>-<seq> form", ids[0])
+	}
+}
+
+func idLess(t *testing.T, a, b string) bool {
+	t.Helper()
+	am, as, err := ParseID(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bm, bs, err := ParseID(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return am < bm || (am == bm && as < bs)
+}
+
+// The watcher's reset clears file:events. The next ID must still exceed every
+// ID issued before the clear, or a consumer holding an old cursor would skip
+// the reset event.
+func TestClearStreamKeepsTheHighWaterMark(t *testing.T) {
+	s := openTest(t, "")
+	var before, after string
+	update(t, s, func(tx *Tx) error {
+		var err error
+		for i := 0; i < 5; i++ {
+			if before, err = tx.Append("file:events", 0, map[string]string{"type": "add"}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	update(t, s, func(tx *Tx) error { return tx.ClearStream("file:events") })
+	if n, _ := s.Len(context.Background(), "file:events"); n != 0 {
+		t.Fatalf("len after clear = %d", n)
+	}
+	update(t, s, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"type": "reset"})
+		return err
+	})
+	if !idLess(t, before, after) {
+		t.Fatalf("post-clear id %s does not follow pre-clear %s", after, before)
+	}
+	got, err := s.ReadAfter(context.Background(), "file:events", before, 10)
+	if err != nil || len(got) != 1 || got[0].Values["type"] != "reset" {
+		t.Fatalf("an old cursor must see the reset: %+v err=%v", got, err)
+	}
+}
+
+// MAXLEN ~ semantics across many small transactions (the common case: one
+// event per write).
+func TestStreamTrimsAcrossTransactions(t *testing.T) {
+	defer func(old int64) { trimEvery = old }(trimEvery)
+	trimEvery = 100
+	s := openTest(t, "")
+	const maxLen = 50
+	for i := int64(0); i < 3*trimEvery; i++ {
+		update(t, s, func(tx *Tx) error {
+			_, err := tx.Append("x", maxLen, map[string]string{"i": "1"})
+			return err
+		})
+	}
+	n, err := s.Len(context.Background(), "x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n > maxLen+trimEvery {
+		t.Fatalf("stream holds %d entries, want <= %d", n, maxLen+trimEvery)
+	}
+	oldest, ok, err := s.Oldest(context.Background(), "x")
+	if err != nil || !ok {
+		t.Fatal(err)
+	}
+	if last, _ := s.LastID(context.Background(), "x"); !idLess(t, oldest.ID, last) {
+		t.Fatalf("oldest %s should precede last %s", oldest.ID, last)
+	}
+}
+
+// A subscriber that took the wake channel before reading is woken by the next
+// commit — the property the SSE pump relies on to not miss an append.
+func TestWaitIsClosedByTheNextCommit(t *testing.T) {
+	s := openTest(t, "")
+	wake := s.Wait("meta:events")
+	update(t, s, func(tx *Tx) error { return tx.Put("file:A/x", "1") })
+	select {
+	case <-wake:
+	case <-time.After(2 * time.Second):
+		t.Fatal("commit did not wake the waiter")
+	}
+}
+
+func TestReadAfterZeroFormsReadFromTheStart(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_, err := tx.Append("x", 0, map[string]string{"n": "1"})
+		return err
+	})
+	for _, id := range []string{"", "0", "0-0"} {
+		got, err := s.ReadAfter(context.Background(), "x", id, 10)
+		if err != nil || len(got) != 1 {
+			t.Errorf("ReadAfter(%q) = %d entries, err %v; want 1", id, len(got), err)
+		}
+	}
+}
+
+func TestRangeIsExactlyThePrefix(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b/c", "file:A0", "file:AB/x", "file:A"} {
+			_ = tx.Put(k, "v")
+		}
+		return nil
+	})
+	rows, err := s.Range(context.Background(), "file:A/", "", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 || rows[0].K != "file:A/a" || rows[1].K != "file:A/b/c" {
+		t.Fatalf("range = %+v", rows)
+	}
+}
+
+// Concurrent Updates share commits; a failing one must roll back alone and
+// leave its neighbours' writes and events intact.
+func TestGroupCommitIsolatesFailures(t *testing.T) {
+	s := openTest(t, "")
+	const n = 200
+	errs := make(chan error, n)
+	for i := 0; i < n; i++ {
+		go func(i int) {
+			errs <- s.Update(context.Background(), func(tx *Tx) error {
+				k := fmt.Sprintf("file:R%03d/title", i)
+				if err := tx.Put(k, "x"); err != nil {
+					return err
+				}
+				if i%10 == 0 {
+					return errors.New("fail")
+				}
+				return nil
+			})
+		}(i)
+	}
+	failed := 0
+	for i := 0; i < n; i++ {
+		if <-errs != nil {
+			failed++
+		}
+	}
+	if failed != n/10 {
+		t.Fatalf("failed = %d, want %d", failed, n/10)
+	}
+	rows, _ := s.Range(context.Background(), "file:", "", 0)
+	if len(rows) != n-n/10 {
+		t.Fatalf("committed %d rows, want %d", len(rows), n-n/10)
+	}
+	if es := events(t, s); len(es) != n-n/10 {
+		t.Fatalf("events = %d, want %d (one per committed write, none for failures)", len(es), n-n/10)
+	}
+	for _, r := range rows {
+		var i int
+		fmt.Sscanf(r.K, "file:R%03d/title", &i)
+		if i%10 == 0 {
+			t.Fatalf("failed request's write %s committed", r.K)
+		}
+	}
+}

internal/kv/stream.go

@@ -0,0 +1,356 @@
+package kv
+
+// Append-only event streams, as rows of the kv table.
+//
+// A stream entry is the row
+//
+//	ev:<stream>/<ms:13>-<seq:10>  →  JSON object of the entry's string fields
+//
+// Zero padding makes key order equal ID order, so reading "after an ID" is a
+// plain range read and trimming is a range delete. On the wire IDs are the
+// unpadded Redis form (`1700000000000-0`): meta-sort, meta-share and
+// meta-stremio persist SSE cursors to disk across the Redis → SQLite switch,
+// and those compare as <ms>-<seq> pairs.
+//
+// The last issued ID is kept in `evhwm:<stream>` and updated in the same
+// transaction as the append. That high-water mark, not the newest surviving
+// row, seeds the next ID — so Clear (the watcher's `reset`) never reissues an
+// ID an old cursor would treat as already seen.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Entry is one stream entry.
+type Entry struct {
+	ID     string
+	Values map[string]string
+}
+
+type streamID struct{ ms, seq int64 }
+
+func (id streamID) String() string {
+	return strconv.FormatInt(id.ms, 10) + "-" + strconv.FormatInt(id.seq, 10)
+}
+
+func (id streamID) padded() string { return fmt.Sprintf("%013d-%010d", id.ms, id.seq) }
+
+func (id streamID) less(o streamID) bool {
+	return id.ms < o.ms || (id.ms == o.ms && id.seq < o.seq)
+}
+
+// ParseID parses a Redis-style stream ID. "", "0", "0-0", "-" all parse as the
+// zero ID (before everything). A bare "<ms>" means "<ms>-0".
+func ParseID(s string) (ms, seq int64, err error) {
+	if s == "" || s == "-" || s == "0" {
+		return 0, 0, nil
+	}
+	msPart, seqPart, hasSeq := strings.Cut(s, "-")
+	ms, err = strconv.ParseInt(msPart, 10, 64)
+	if err != nil || ms < 0 {
+		return 0, 0, fmt.Errorf("invalid stream id %q", s)
+	}
+	if hasSeq {
+		seq, err = strconv.ParseInt(seqPart, 10, 64)
+		if err != nil || seq < 0 {
+			return 0, 0, fmt.Errorf("invalid stream id %q", s)
+		}
+	}
+	return ms, seq, nil
+}
+
+func streamPrefix(stream string) string { return "ev:" + stream + "/" }
+func hwmKey(stream string) string       { return "evhwm:" + stream }
+
+// streamState is the in-memory view of a stream's high-water mark plus the
+// broadcast channel subscribers wait on. Updated only after a commit.
+type streamState struct {
+	loaded    bool
+	hwm       streamID
+	sinceTrim int64 // committed appends since the last trim
+	notify    chan struct{}
+}
+
+// streamPending is what a transaction did to one stream, applied to
+// streamState on commit (and discarded on rollback).
+type streamPending struct {
+	hwm        streamID
+	appends    int64
+	trimmed    bool
+	lastTrimAt int64 // value of appends when this transaction last trimmed
+}
+
+func (s *Store) state(stream string) *streamState {
+	st := s.streams[stream]
+	if st == nil {
+		st = &streamState{notify: make(chan struct{})}
+		s.streams[stream] = st
+	}
+	return st
+}
+
+// Wait returns a channel closed by the next commit that appends to stream.
+// Take it BEFORE reading, so an append between the read and the wait is not
+// missed.
+func (s *Store) Wait(stream string) <-chan struct{} {
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	return s.state(stream).notify
+}
+
+func (s *Store) afterCommit(pending map[string]*streamPending) {
+	if len(pending) == 0 {
+		return
+	}
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	for name, p := range pending {
+		st := s.state(name)
+		st.loaded = true
+		st.hwm = p.hwm
+		if p.trimmed {
+			st.sinceTrim = p.appends - p.lastTrimAt
+		} else {
+			st.sinceTrim += p.appends
+		}
+		close(st.notify)
+		st.notify = make(chan struct{})
+	}
+}
+
+// hwm returns the stream's last issued ID as seen by this transaction.
+func (t *Tx) hwm(stream string) (streamID, error) {
+	if p := t.g.pending[stream]; p != nil {
+		return p.hwm, nil
+	}
+	t.s.streamMu.Lock()
+	st := t.s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	t.s.streamMu.Unlock()
+	if loaded {
+		return hwm, nil
+	}
+	v, ok, err := t.Get(t.ctx, hwmKey(stream))
+	if err != nil {
+		return streamID{}, err
+	}
+	if !ok {
+		return streamID{}, nil
+	}
+	ms, seq, err := ParseID(v)
+	if err != nil {
+		return streamID{}, err
+	}
+	return streamID{ms, seq}, nil
+}
+
+// Append adds an entry to stream and returns its ID. The stream is trimmed to
+// about maxLen entries (maxLen <= 0: unbounded).
+func (t *Tx) Append(stream string, maxLen int64, fields map[string]string) (string, error) {
+	last, err := t.hwm(stream)
+	if err != nil {
+		return "", fmt.Errorf("stream %s hwm: %w", stream, err)
+	}
+	id := streamID{ms: time.Now().UnixMilli()}
+	if !last.less(id) {
+		id = streamID{ms: last.ms, seq: last.seq + 1}
+	}
+	body, err := json.Marshal(fields)
+	if err != nil {
+		return "", err
+	}
+	if _, err := t.stmt(t.s.stInsert).ExecContext(t.ctx, streamPrefix(stream)+id.padded(), string(body)); err != nil {
+		return "", fmt.Errorf("append %s: %w", stream, err)
+	}
+	if t.g.pending == nil {
+		t.g.pending = make(map[string]*streamPending)
+	}
+	p := t.g.pending[stream]
+	if p == nil {
+		p = &streamPending{}
+		t.g.pending[stream] = p
+	}
+	p.hwm = id
+	p.appends++
+	// Trim in steps, not per append: MAXLEN ~ semantics, like the XADD it
+	// replaces. The count spans transactions (most append one event each).
+	if maxLen > 0 {
+		t.s.streamMu.Lock()
+		since := t.s.state(stream).sinceTrim
+		t.s.streamMu.Unlock()
+		if !p.trimmed {
+			since += p.appends
+		} else {
+			since = p.appends - p.lastTrimAt
+		}
+		if since >= trimEvery {
+			if err := t.trim(stream, maxLen); err != nil {
+				return "", err
+			}
+			p.trimmed, p.lastTrimAt = true, p.appends
+		}
+	}
+	return id.String(), nil
+}
+
+// trimEvery is how many appends pass between trims. A trim locates its cut
+// point by walking maxLen entries back from the newest, so it is kept rare;
+// the cost is overshooting maxLen by at most trimEvery entries.
+var trimEvery int64 = 10_000 // var so tests can lower it
+
+func (t *Tx) trim(stream string, maxLen int64) error {
+	prefix := streamPrefix(stream)
+	var cut string
+	err := t.sqlTx().QueryRowContext(t.ctx,
+		`SELECT k FROM kv WHERE k >= ? AND k < ? ORDER BY k DESC LIMIT 1 OFFSET ?`,
+		prefix, PrefixEnd(prefix), maxLen).Scan(&cut)
+	if err != nil {
+		return nil // fewer than maxLen entries (sql.ErrNoRows): nothing to trim
+	}
+	_, err = t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k <= ?`, prefix, cut)
+	return err
+}
+
+// flushHWM persists each touched stream's high-water mark — once per
+// transaction, not per append.
+func (t *Tx) flushHWM() error {
+	for stream, p := range t.g.pending {
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, hwmKey(stream), p.hwm.String()); err != nil {
+			return fmt.Errorf("stream %s hwm: %w", stream, err)
+		}
+	}
+	return nil
+}
+
+// TrimAll trims stream to maxLen inside the transaction regardless of how many
+// appends this transaction made. Used after an append burst split over many
+// short transactions.
+func (t *Tx) TrimAll(stream string, maxLen int64) error { return t.trim(stream, maxLen) }
+
+// ClearStream deletes every entry of stream, keeping its high-water mark.
+func (t *Tx) ClearStream(stream string) error {
+	prefix := streamPrefix(stream)
+	_, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k < ?`, prefix, PrefixEnd(prefix))
+	return err
+}
+
+func decodeEntry(stream, k, v string) (Entry, error) {
+	padded := strings.TrimPrefix(k, streamPrefix(stream))
+	ms, seq, err := ParseID(padded)
+	if err != nil {
+		return Entry{}, err
+	}
+	var values map[string]string
+	if err := json.Unmarshal([]byte(v), &values); err != nil {
+		return Entry{}, fmt.Errorf("decode %s: %w", k, err)
+	}
+	return Entry{ID: streamID{ms, seq}.String(), Values: values}, nil
+}
+
+func decodeEntries(stream string, rows []KV) ([]Entry, error) {
+	out := make([]Entry, 0, len(rows))
+	for _, kv := range rows {
+		e, err := decodeEntry(stream, kv.K, kv.V)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, e)
+	}
+	return out, nil
+}
+
+// ReadAfter returns up to n entries strictly after id, oldest first. "0",
+// "0-0" and "" read from the beginning.
+func (s *Store) ReadAfter(ctx context.Context, stream, id string, n int) ([]Entry, error) {
+	ms, seq, err := ParseID(id)
+	if err != nil {
+		return nil, err
+	}
+	prefix := streamPrefix(stream)
+	after := ""
+	if ms != 0 || seq != 0 {
+		after = prefix + streamID{ms, seq}.padded()
+	}
+	rows, err := rangeQuery(ctx, s.r, prefix, after, n)
+	if err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, rows)
+}
+
+// Oldest returns the oldest surviving entry, or ok=false for an empty stream.
+func (s *Store) Oldest(ctx context.Context, stream string) (Entry, bool, error) {
+	rows, err := rangeQuery(ctx, s.r, streamPrefix(stream), "", 1)
+	if err != nil || len(rows) == 0 {
+		return Entry{}, false, err
+	}
+	e, err := decodeEntry(stream, rows[0].K, rows[0].V)
+	return e, err == nil, err
+}
+
+// LastID returns the last ID ever issued on stream ("0-0" if none) — the
+// cursor meaning "only entries appended from now on".
+func (s *Store) LastID(ctx context.Context, stream string) (string, error) {
+	s.streamMu.Lock()
+	st := s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	s.streamMu.Unlock()
+	if loaded {
+		return hwm.String(), nil
+	}
+	v, ok, err := s.Get(ctx, hwmKey(stream))
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "0-0", nil
+	}
+	return v, nil
+}
+
+// Len is the number of surviving entries.
+func (s *Store) Len(ctx context.Context, stream string) (int64, error) {
+	return count(ctx, s.r, streamPrefix(stream))
+}
+
+// RevRange returns up to n entries newest first, stopping at (excluding)
+// entries at or before afterID when afterID is non-empty.
+func (s *Store) RevRange(ctx context.Context, stream, afterID string, n int) ([]Entry, error) {
+	prefix := streamPrefix(stream)
+	query := `SELECT k, v FROM kv WHERE k > ? AND k < ? ORDER BY k DESC`
+	lo := prefix
+	if afterID != "" {
+		ms, seq, err := ParseID(afterID)
+		if err != nil {
+			return nil, err
+		}
+		lo = prefix + streamID{ms, seq}.padded()
+	}
+	args := []any{lo, PrefixEnd(prefix)}
+	if n > 0 {
+		query += ` LIMIT ?`
+		args = append(args, n)
+	}
+	rows, err := s.r.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var kvs []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		kvs = append(kvs, kv)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, kvs)
+}