v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/kv/kv.go

@@ -53,8 +53,10 @@ import (
 	"log"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	_ "modernc.org/sqlite"
@@ -77,6 +79,11 @@ type Options struct {
 	// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
 	// for derived indexes owned by the caller.
 	Schema string
+	// OnChange is called inside the transaction for every key whose value
+	// actually changed (created, modified or deleted), with the old and new
+	// values — the hook derived counters hang off (Tx.AddCounter). It must not
+	// write kv rows itself.
+	OnChange func(tx *Tx, key, old string, hadOld bool, new string, hasNew bool)
 }
 
 // Store is an open database.
@@ -93,6 +100,9 @@ type Store struct {
 	// SQL on every Put was the single largest cost of a record write.
 	stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
 
+	// Write accounting: how many Puts changed a value vs found it unchanged.
+	puts, noops atomic.Int64
+
 	writes    chan *writeReq
 	writeMu   sync.Mutex // held by the writer while a group runs
 	closeOnce sync.Once
@@ -170,7 +180,7 @@ func Open(path string, opts Options) (*Store, error) {
 	}{
 		{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
 		{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
-		{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
+		{&st.stDelete, `DELETE FROM kv WHERE k = ? RETURNING v`},
 		{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
 		{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
 	} {
@@ -470,9 +480,10 @@ func (s *Store) writer() {
 
 // group is the state shared by the requests of one commit.
 type group struct {
-	tx      *sql.Tx
-	pending map[string]*streamPending
-	stmts   map[*sql.Stmt]*sql.Stmt
+	tx       *sql.Tx
+	pending  map[string]*streamPending
+	stmts    map[*sql.Stmt]*sql.Stmt
+	counters map[string]int64
 }
 
 func (g *group) snapshot() map[string]streamPending {
@@ -521,6 +532,9 @@ func (s *Store) runGroup(reqs []*writeReq) {
 		if err == nil {
 			err = tx.flushEvents()
 		}
+		if err == nil {
+			g.mergeCounters(tx.counters)
+		}
 		if err != nil {
 			results[i] = err
 			if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
@@ -536,6 +550,11 @@ func (s *Store) runGroup(reqs []*writeReq) {
 			return
 		}
 	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushCounters(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
 	if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
 		_ = sqlTx.Rollback()
 		fail(err)
@@ -554,10 +573,11 @@ func (s *Store) runGroup(reqs []*writeReq) {
 // Tx is one Update's view of the write transaction. Its reads see its own
 // writes (and those of earlier requests in the same group).
 type Tx struct {
-	ctx    context.Context
-	s      *Store
-	g      *group
-	events []map[string]string
+	ctx      context.Context
+	s        *Store
+	g        *group
+	events   []map[string]string
+	counters map[string]int64
 }
 
 func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
@@ -610,18 +630,34 @@ func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
 	return t.sqlTx().QueryRowContext(t.ctx, query, args...)
 }
 
-// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
-// keyspace notifications did.
+// Put sets k = v. A no-op — no row write, no index churn, no event — when k
+// already holds v. (Redis keyspace notifications published `set` even for an
+// unchanged value; with meta-core persisting the same gateway hits again on
+// every repeated query, those no-op writes were most of the write load.)
 func (t *Tx) Put(k, v string) error {
-	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
-		return fmt.Errorf("put %q: %w", k, err)
+	_, err := t.Set(k, v)
+	return err
+}
+
+// Set is Put that reports whether the value changed.
+func (t *Tx) Set(k, v string) (bool, error) {
+	old, had, err := t.Get(t.ctx, k)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
 	}
-	t.event("set", k)
-	return nil
+	if had && old == v {
+		t.s.noops.Add(1)
+		return false, nil
+	}
+	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	t.s.puts.Add(1)
+	t.changed("set", k, old, had, v, true)
+	return true, nil
 }
 
 // PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
-// Publishes nothing when the key already existed (nothing changed).
 func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 	res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
 	if err != nil {
@@ -629,7 +665,7 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 	}
 	n, _ := res.RowsAffected()
 	if n > 0 {
-		t.event("set", k)
+		t.changed("set", k, "", false, v, true)
 	}
 	return n > 0, nil
 }
@@ -637,15 +673,16 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 // Delete removes k. Reports whether it existed; publishes `del` only then,
 // as Redis did.
 func (t *Tx) Delete(k string) (bool, error) {
-	res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
+	var old string
+	err := t.stmt(t.s.stDelete).QueryRowContext(t.ctx, k).Scan(&old)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
 	if err != nil {
 		return false, fmt.Errorf("delete %q: %w", k, err)
 	}
-	n, _ := res.RowsAffected()
-	if n > 0 {
-		t.event("del", k)
-	}
-	return n > 0, nil
+	t.changed("del", k, old, true, "", false)
+	return true, nil
 }
 
 // DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
@@ -663,26 +700,103 @@ func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
 		sel += ` ORDER BY k LIMIT ?`
 		args = append(args, limit)
 	}
-	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
+	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k, v`, args...)
 	if err != nil {
 		return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
 	}
-	var keys []string
+	var gone []KV
 	for rows.Next() {
-		var k string
-		if err := rows.Scan(&k); err != nil {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
 			rows.Close()
 			return 0, err
 		}
-		keys = append(keys, k)
+		gone = append(gone, kv)
 	}
 	if err := rows.Close(); err != nil {
 		return 0, err
 	}
-	for _, k := range keys {
-		t.event("del", k)
+	for _, kv := range gone {
+		t.changed("del", kv.K, kv.V, true, "", false)
 	}
-	return int64(len(keys)), nil
+	return int64(len(gone)), nil
+}
+
+// changed records one real change: the event (when in scope) and the hook.
+func (t *Tx) changed(op, k, old string, hadOld bool, v string, hasNew bool) {
+	t.event(op, k)
+	if t.s.opts.OnChange != nil {
+		t.s.opts.OnChange(t, k, old, hadOld, v, hasNew)
+	}
+}
+
+// AddCounter adds delta to the integer counter stored at key. Deltas are
+// summed in memory and written once per commit (a hot counter touched by every
+// record in a group costs one row write, not one per record); they are
+// discarded with the request if it fails. A counter that reaches zero is
+// deleted.
+func (t *Tx) AddCounter(key string, delta int64) {
+	if delta == 0 {
+		return
+	}
+	if t.counters == nil {
+		t.counters = make(map[string]int64)
+	}
+	t.counters[key] += delta
+}
+
+func (g *group) mergeCounters(c map[string]int64) {
+	if len(c) == 0 {
+		return
+	}
+	if g.counters == nil {
+		g.counters = make(map[string]int64, len(c))
+	}
+	for k, d := range c {
+		g.counters[k] += d
+	}
+}
+
+// flushCounters applies the group's counter deltas, bypassing Put (counters
+// are bookkeeping: no events, no hook).
+func (t *Tx) flushCounters() error {
+	for k, d := range t.g.counters {
+		if d == 0 {
+			continue
+		}
+		cur, _, err := t.Get(t.ctx, k)
+		if err != nil {
+			return err
+		}
+		n, _ := strconv.ParseInt(cur, 10, 64)
+		n += d
+		if n <= 0 {
+			if _, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k = ?`, k); err != nil {
+				return fmt.Errorf("counter %s: %w", k, err)
+			}
+			continue
+		}
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, strconv.FormatInt(n, 10)); err != nil {
+			return fmt.Errorf("counter %s: %w", k, err)
+		}
+	}
+	return nil
+}
+
+// Counter reads a committed counter (0 when absent).
+func (s *Store) Counter(ctx context.Context, key string) (int64, bool, error) {
+	v, ok, err := s.Get(ctx, key)
+	if err != nil || !ok {
+		return 0, ok, err
+	}
+	n, err := strconv.ParseInt(v, 10, 64)
+	return n, true, err
+}
+
+// WriteStats reports how many Puts changed a value and how many found it
+// already in place, since open.
+func (s *Store) WriteStats() (changed, unchanged int64) {
+	return s.puts.Load(), s.noops.Load()
 }
 
 func (t *Tx) event(op, k string) {