v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/kv/kv.go

@@ -53,8 +53,10 @@ import (
 	"log"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	_ "modernc.org/sqlite"
@@ -77,6 +79,11 @@ type Options struct {
 	// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
 	// for derived indexes owned by the caller.
 	Schema string
+	// OnChange is called inside the transaction for every key whose value
+	// actually changed (created, modified or deleted), with the old and new
+	// values — the hook derived counters hang off (Tx.AddCounter). It must not
+	// write kv rows itself.
+	OnChange func(tx *Tx, key, old string, hadOld bool, new string, hasNew bool)
 }
 
 // Store is an open database.
@@ -93,6 +100,9 @@ type Store struct {
 	// SQL on every Put was the single largest cost of a record write.
 	stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
 
+	// Write accounting: how many Puts changed a value vs found it unchanged.
+	puts, noops atomic.Int64
+
 	writes    chan *writeReq
 	writeMu   sync.Mutex // held by the writer while a group runs
 	closeOnce sync.Once
@@ -170,7 +180,7 @@ func Open(path string, opts Options) (*Store, error) {
 	}{
 		{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
 		{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
-		{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
+		{&st.stDelete, `DELETE FROM kv WHERE k = ? RETURNING v`},
 		{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
 		{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
 	} {
@@ -470,9 +480,10 @@ func (s *Store) writer() {
 
 // group is the state shared by the requests of one commit.
 type group struct {
-	tx      *sql.Tx
-	pending map[string]*streamPending
-	stmts   map[*sql.Stmt]*sql.Stmt
+	tx       *sql.Tx
+	pending  map[string]*streamPending
+	stmts    map[*sql.Stmt]*sql.Stmt
+	counters map[string]int64
 }
 
 func (g *group) snapshot() map[string]streamPending {
@@ -521,6 +532,9 @@ func (s *Store) runGroup(reqs []*writeReq) {
 		if err == nil {
 			err = tx.flushEvents()
 		}
+		if err == nil {
+			g.mergeCounters(tx.counters)
+		}
 		if err != nil {
 			results[i] = err
 			if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
@@ -536,6 +550,11 @@ func (s *Store) runGroup(reqs []*writeReq) {
 			return
 		}
 	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushCounters(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
 	if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
 		_ = sqlTx.Rollback()
 		fail(err)
@@ -554,10 +573,11 @@ func (s *Store) runGroup(reqs []*writeReq) {
 // Tx is one Update's view of the write transaction. Its reads see its own
 // writes (and those of earlier requests in the same group).
 type Tx struct {
-	ctx    context.Context
-	s      *Store
-	g      *group
-	events []map[string]string
+	ctx      context.Context
+	s        *Store
+	g        *group
+	events   []map[string]string
+	counters map[string]int64
 }
 
 func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
@@ -610,18 +630,34 @@ func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
 	return t.sqlTx().QueryRowContext(t.ctx, query, args...)
 }
 
-// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
-// keyspace notifications did.
+// Put sets k = v. A no-op — no row write, no index churn, no event — when k
+// already holds v. (Redis keyspace notifications published `set` even for an
+// unchanged value; with meta-core persisting the same gateway hits again on
+// every repeated query, those no-op writes were most of the write load.)
 func (t *Tx) Put(k, v string) error {
-	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
-		return fmt.Errorf("put %q: %w", k, err)
+	_, err := t.Set(k, v)
+	return err
+}
+
+// Set is Put that reports whether the value changed.
+func (t *Tx) Set(k, v string) (bool, error) {
+	old, had, err := t.Get(t.ctx, k)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
 	}
-	t.event("set", k)
-	return nil
+	if had && old == v {
+		t.s.noops.Add(1)
+		return false, nil
+	}
+	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	t.s.puts.Add(1)
+	t.changed("set", k, old, had, v, true)
+	return true, nil
 }
 
 // PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
-// Publishes nothing when the key already existed (nothing changed).
 func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 	res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
 	if err != nil {
@@ -629,7 +665,7 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 	}
 	n, _ := res.RowsAffected()
 	if n > 0 {
-		t.event("set", k)
+		t.changed("set", k, "", false, v, true)
 	}
 	return n > 0, nil
 }
@@ -637,15 +673,16 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
 // Delete removes k. Reports whether it existed; publishes `del` only then,
 // as Redis did.
 func (t *Tx) Delete(k string) (bool, error) {
-	res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
+	var old string
+	err := t.stmt(t.s.stDelete).QueryRowContext(t.ctx, k).Scan(&old)
+	if errors.Is(err, sql.ErrNoRows) {
+		return false, nil
+	}
 	if err != nil {
 		return false, fmt.Errorf("delete %q: %w", k, err)
 	}
-	n, _ := res.RowsAffected()
-	if n > 0 {
-		t.event("del", k)
-	}
-	return n > 0, nil
+	t.changed("del", k, old, true, "", false)
+	return true, nil
 }
 
 // DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
@@ -663,26 +700,103 @@ func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
 		sel += ` ORDER BY k LIMIT ?`
 		args = append(args, limit)
 	}
-	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
+	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k, v`, args...)
 	if err != nil {
 		return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
 	}
-	var keys []string
+	var gone []KV
 	for rows.Next() {
-		var k string
-		if err := rows.Scan(&k); err != nil {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
 			rows.Close()
 			return 0, err
 		}
-		keys = append(keys, k)
+		gone = append(gone, kv)
 	}
 	if err := rows.Close(); err != nil {
 		return 0, err
 	}
-	for _, k := range keys {
-		t.event("del", k)
+	for _, kv := range gone {
+		t.changed("del", kv.K, kv.V, true, "", false)
 	}
-	return int64(len(keys)), nil
+	return int64(len(gone)), nil
+}
+
+// changed records one real change: the event (when in scope) and the hook.
+func (t *Tx) changed(op, k, old string, hadOld bool, v string, hasNew bool) {
+	t.event(op, k)
+	if t.s.opts.OnChange != nil {
+		t.s.opts.OnChange(t, k, old, hadOld, v, hasNew)
+	}
+}
+
+// AddCounter adds delta to the integer counter stored at key. Deltas are
+// summed in memory and written once per commit (a hot counter touched by every
+// record in a group costs one row write, not one per record); they are
+// discarded with the request if it fails. A counter that reaches zero is
+// deleted.
+func (t *Tx) AddCounter(key string, delta int64) {
+	if delta == 0 {
+		return
+	}
+	if t.counters == nil {
+		t.counters = make(map[string]int64)
+	}
+	t.counters[key] += delta
+}
+
+func (g *group) mergeCounters(c map[string]int64) {
+	if len(c) == 0 {
+		return
+	}
+	if g.counters == nil {
+		g.counters = make(map[string]int64, len(c))
+	}
+	for k, d := range c {
+		g.counters[k] += d
+	}
+}
+
+// flushCounters applies the group's counter deltas, bypassing Put (counters
+// are bookkeeping: no events, no hook).
+func (t *Tx) flushCounters() error {
+	for k, d := range t.g.counters {
+		if d == 0 {
+			continue
+		}
+		cur, _, err := t.Get(t.ctx, k)
+		if err != nil {
+			return err
+		}
+		n, _ := strconv.ParseInt(cur, 10, 64)
+		n += d
+		if n <= 0 {
+			if _, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k = ?`, k); err != nil {
+				return fmt.Errorf("counter %s: %w", k, err)
+			}
+			continue
+		}
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, strconv.FormatInt(n, 10)); err != nil {
+			return fmt.Errorf("counter %s: %w", k, err)
+		}
+	}
+	return nil
+}
+
+// Counter reads a committed counter (0 when absent).
+func (s *Store) Counter(ctx context.Context, key string) (int64, bool, error) {
+	v, ok, err := s.Get(ctx, key)
+	if err != nil || !ok {
+		return 0, ok, err
+	}
+	n, err := strconv.ParseInt(v, 10, 64)
+	return n, true, err
+}
+
+// WriteStats reports how many Puts changed a value and how many found it
+// already in place, since open.
+func (s *Store) WriteStats() (changed, unchanged int64) {
+	return s.puts.Load(), s.noops.Load()
 }
 
 func (t *Tx) event(op, k string) {

internal/kv/kv_test.go

@@ -44,20 +44,21 @@ func events(t *testing.T, s *Store) []Entry {
 }
 
 // The replacement for keyspace notifications: a change and its event commit
-// together, with Redis' semantics — `set` even for an unchanged value, `del`
-// only for a key that existed, and nothing for keys outside the event scope.
+// together. `set` only for a real change (an unchanged value is a no-op: no
+// write, no event), `del` only for a key that existed, nothing for keys outside
+// the event scope.
 func TestChangesPublishEventsWithRedisSemantics(t *testing.T) {
 	s := openTest(t, "")
 	update(t, s, func(tx *Tx) error {
 		_ = tx.Put("file:A/title", "x")
-		_ = tx.Put("file:A/title", "x") // unchanged value still publishes
+		_ = tx.Put("file:A/title", "x") // unchanged: no write, no event
 		_ = tx.Put("cid:bafk", "A")     // outside the event scope
 		_, _ = tx.Delete("file:A/missing")
 		_, _ = tx.Delete("file:A/title")
 		return nil
 	})
 	got := events(t, s)
-	want := []string{"set file:A/title", "set file:A/title", "del file:A/title"}
+	want := []string{"set file:A/title", "del file:A/title"}
 	if len(got) != len(want) {
 		t.Fatalf("events = %+v, want %v", got, want)
 	}
@@ -312,3 +313,50 @@ func TestGroupCommitIsolatesFailures(t *testing.T) {
 		}
 	}
 }
+
+// The change hook sees real changes only, with old and new values, and
+// counters follow the request's fate (a failed request's deltas are dropped).
+func TestOnChangeAndCounters(t *testing.T) {
+	type change struct{ k, old, new string }
+	var seen []change
+	s, err := Open(filepath.Join(t.TempDir(), "kv.db"), Options{
+		OnChange: func(tx *Tx, k, old string, hadOld bool, v string, hasNew bool) {
+			seen = append(seen, change{k, old, v})
+			if hasNew {
+				tx.AddCounter("count/"+v, 1)
+			}
+			if hadOld {
+				tx.AddCounter("count/"+old, -1)
+			}
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	update(t, s, func(tx *Tx) error {
+		_ = tx.Put("a", "x")
+		_ = tx.Put("b", "x")
+		_ = tx.Put("a", "x") // no-op
+		_ = tx.Put("a", "y")
+		return nil
+	})
+	_ = s.Update(context.Background(), func(tx *Tx) error {
+		_ = tx.Put("c", "x")
+		return errors.New("fail")
+	})
+	update(t, s, func(tx *Tx) error { _, err := tx.DeletePrefix("b", 0); return err })
+
+	if len(seen) != 5 { // a=x, b=x, a x→y, c=x (rolled back, hook still ran), del b; stats count attempts
+		t.Fatalf("changes = %+v", seen)
+	}
+	for k, want := range map[string]int64{"count/x": 0, "count/y": 1} {
+		got, _, err := s.Counter(context.Background(), k)
+		if err != nil || got != want {
+			t.Errorf("%s = %d (err %v), want %d", k, got, err, want)
+		}
+	}
+	if changed, unchanged := s.WriteStats(); changed != 4 || unchanged != 1 {
+		t.Errorf("write stats = %d changed / %d unchanged", changed, unchanged)
+	}
+}