replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
Build and Push Docker Image / build (push) Successful in 2m44s
This commit is contained in:
Worph
@@ -0,0 +1,713 @@
|
||||
// Package kv is meta-core's storage engine: one string→string table in an
|
||||
// embedded SQLite file.
|
||||
//
|
||||
// # Why a key/value table and not a schema
|
||||
//
|
||||
// meta-core's data model is schema-free on purpose — a record is a set of flat
|
||||
// `file:<rid>/<field>` keys, and a new metadata field is just a new key
|
||||
// (METADATA_KEYS.md). The engine keeps exactly that model:
|
||||
//
|
||||
// CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID
|
||||
//
|
||||
// is the only table that holds data, and it never needs a migration. What
|
||||
// Redis gave us that a bare KV would not — "list a record's fields", "find
|
||||
// records by field value", free-text search — comes from the table being
|
||||
// *ordered* (a record is a key range) and from two derived structures the
|
||||
// storage package maintains on top: an expression index over the field name
|
||||
// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv
|
||||
// rows and never authoritative.
|
||||
//
|
||||
// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a
|
||||
// 169k-record corpus (Redis plus the API's in-memory copy needed to search
|
||||
// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here
|
||||
// and the working set is a bounded page cache.
|
||||
//
|
||||
// # Concurrency
|
||||
//
|
||||
// SQLite allows one writer at a time, so every write goes through one writer
|
||||
// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED:
|
||||
// the writer runs every request waiting at that moment inside one SQLite
|
||||
// transaction, each under its own SAVEPOINT (a failing request rolls back
|
||||
// alone), and commits once. A commit — writing the dirty pages of the table,
|
||||
// the field index and the FTS shadow tables to the WAL — was ~40% of the cost
|
||||
// of a record write, so sharing it is what makes concurrent ingest scale.
|
||||
//
|
||||
// Reads use a separate pool and see the last committed state (WAL). Inside
|
||||
// Update, read through the *Tx — the reader pool cannot see uncommitted rows —
|
||||
// and never call Update from inside an Update callback (the writer would wait
|
||||
// for itself).
|
||||
//
|
||||
// # Events
|
||||
//
|
||||
// Redis keyspace notifications are replaced by the transaction itself: every
|
||||
// Put/Delete of a key the Options.EventKey predicate selects is appended, in
|
||||
// the same transaction, to Options.EventStream as {type: set|del, key, ts}.
|
||||
// A change and its event therefore commit together or not at all.
|
||||
package kv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Options configures a Store.
|
||||
type Options struct {
|
||||
// EventStream receives a set/del entry for every changed key EventKey
|
||||
// selects. Empty disables change events.
|
||||
EventStream string
|
||||
// EventMaxLen is the approximate retention of EventStream.
|
||||
EventMaxLen int64
|
||||
// EventKey selects the keys whose changes are published.
|
||||
EventKey func(key string) bool
|
||||
// WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection.
|
||||
WriterCacheMB int
|
||||
ReaderCacheMB int
|
||||
// Readers is the size of the read connection pool.
|
||||
Readers int
|
||||
// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
|
||||
// for derived indexes owned by the caller.
|
||||
Schema string
|
||||
}
|
||||
|
||||
// Store is an open database.
|
||||
type Store struct {
|
||||
path string
|
||||
opts Options
|
||||
w *sql.DB // single connection: the one writer
|
||||
r *sql.DB // read pool
|
||||
|
||||
streamMu sync.Mutex
|
||||
streams map[string]*streamState
|
||||
|
||||
// Hot write statements, prepared once on the writer connection: parsing
|
||||
// SQL on every Put was the single largest cost of a record write.
|
||||
stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
|
||||
|
||||
writes chan *writeReq
|
||||
writeMu sync.Mutex // held by the writer while a group runs
|
||||
closeOnce sync.Once
|
||||
stopped chan struct{}
|
||||
}
|
||||
|
||||
// writeReq is one Update call waiting for the writer.
|
||||
type writeReq struct {
|
||||
fn func(tx *Tx) error
|
||||
done chan error
|
||||
}
|
||||
|
||||
// maxGroup caps how many requests share one commit, bounding how long the
|
||||
// writer holds a transaction open.
|
||||
const maxGroup = 128
|
||||
|
||||
// Open opens (creating if needed) the database file at path.
|
||||
func Open(path string, opts Options) (*Store, error) {
|
||||
if opts.Readers <= 0 {
|
||||
opts.Readers = 8
|
||||
}
|
||||
if opts.WriterCacheMB <= 0 {
|
||||
opts.WriterCacheMB = 64
|
||||
}
|
||||
if opts.ReaderCacheMB <= 0 {
|
||||
opts.ReaderCacheMB = 16
|
||||
}
|
||||
if opts.EventMaxLen <= 0 {
|
||||
opts.EventMaxLen = 100_000
|
||||
}
|
||||
|
||||
w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open writer: %w", err)
|
||||
}
|
||||
w.SetMaxOpenConns(1)
|
||||
w.SetMaxIdleConns(1)
|
||||
w.SetConnMaxLifetime(0)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
|
||||
w.Close()
|
||||
return nil, fmt.Errorf("create kv: %w", err)
|
||||
}
|
||||
if opts.Schema != "" {
|
||||
// No deadline: on an existing store, the first open after a new index
|
||||
// is added builds it over every row (minutes at millions of keys).
|
||||
// Cancelling that just rolls it back and restarts it on the next boot.
|
||||
started := time.Now()
|
||||
if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil {
|
||||
w.Close()
|
||||
return nil, fmt.Errorf("apply schema: %w", err)
|
||||
}
|
||||
if d := time.Since(started); d > 5*time.Second {
|
||||
log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second))
|
||||
}
|
||||
}
|
||||
|
||||
r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true))
|
||||
if err != nil {
|
||||
w.Close()
|
||||
return nil, fmt.Errorf("open readers: %w", err)
|
||||
}
|
||||
r.SetMaxOpenConns(opts.Readers)
|
||||
r.SetMaxIdleConns(opts.Readers)
|
||||
|
||||
st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)}
|
||||
// Fresh deadline: the schema step above may legitimately have taken minutes.
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
for _, p := range []struct {
|
||||
dst **sql.Stmt
|
||||
query string
|
||||
}{
|
||||
{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
|
||||
{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
|
||||
{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
|
||||
{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
|
||||
{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
|
||||
} {
|
||||
if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil {
|
||||
r.Close()
|
||||
w.Close()
|
||||
return nil, fmt.Errorf("prepare: %w", err)
|
||||
}
|
||||
}
|
||||
st.writes = make(chan *writeReq)
|
||||
st.stopped = make(chan struct{})
|
||||
go st.writer()
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class
|
||||
// Redis' AOF-everysec gave us: a power loss may drop the last transactions,
|
||||
// never corrupt the file.
|
||||
func dsn(path string, cacheMB int, readOnly bool) string {
|
||||
q := url.Values{}
|
||||
q.Add("_pragma", "busy_timeout(30000)")
|
||||
q.Add("_pragma", "journal_mode(WAL)")
|
||||
q.Add("_pragma", "synchronous(NORMAL)")
|
||||
q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024))
|
||||
q.Add("_pragma", "temp_store(MEMORY)")
|
||||
q.Add("_pragma", "journal_size_limit(268435456)")
|
||||
if readOnly {
|
||||
q.Add("_pragma", "query_only(1)")
|
||||
} else {
|
||||
q.Add("_txlock", "immediate")
|
||||
}
|
||||
return "file:" + path + "?" + q.Encode()
|
||||
}
|
||||
|
||||
// Close stops the writer and closes both pools, checkpointing the WAL into the
|
||||
// main file.
|
||||
func (s *Store) Close() error {
|
||||
s.closeOnce.Do(func() { close(s.stopped) })
|
||||
s.writeMu.Lock() // wait for an in-flight group to finish
|
||||
defer s.writeMu.Unlock()
|
||||
_, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`)
|
||||
return errors.Join(s.r.Close(), s.w.Close())
|
||||
}
|
||||
|
||||
// Ping reports whether the database answers.
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
var one int
|
||||
return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one)
|
||||
}
|
||||
|
||||
// Path is the database file path.
|
||||
func (s *Store) Path() string { return s.path }
|
||||
|
||||
// DB exposes the read pool for callers that own derived indexes (search). It
|
||||
// is read-only (query_only), so it cannot bypass Update.
|
||||
func (s *Store) DB() *sql.DB { return s.r }
|
||||
|
||||
// FileSize returns the on-disk size of the database plus its WAL.
|
||||
func (s *Store) FileSize() int64 {
|
||||
var total int64
|
||||
for _, suffix := range []string{"", "-wal"} {
|
||||
if fi, err := os.Stat(s.path + suffix); err == nil {
|
||||
total += fi.Size()
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer
|
||||
// slot and waits for readers, so callers run it when idle.
|
||||
func (s *Store) Checkpoint(ctx context.Context) error {
|
||||
_, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
|
||||
return err
|
||||
}
|
||||
|
||||
// KV is one key/value pair.
|
||||
type KV struct {
|
||||
K, V string
|
||||
}
|
||||
|
||||
// Reader is the read surface shared by Store (committed state) and Tx (the
|
||||
// transaction's own view). Storage helpers take a Reader so they work both
|
||||
// inside and outside a write.
|
||||
type Reader interface {
|
||||
Get(ctx context.Context, k string) (string, bool, error)
|
||||
MGet(ctx context.Context, keys []string) (map[string]string, error)
|
||||
Range(ctx context.Context, prefix, after string, limit int) ([]KV, error)
|
||||
Count(ctx context.Context, prefix string) (int64, error)
|
||||
}
|
||||
|
||||
type queryer interface {
|
||||
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
|
||||
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
|
||||
}
|
||||
|
||||
// PrefixEnd returns the smallest string greater than every string with the
|
||||
// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range.
|
||||
// An empty result means "unbounded".
|
||||
func PrefixEnd(prefix string) string {
|
||||
b := []byte(prefix)
|
||||
for i := len(b) - 1; i >= 0; i-- {
|
||||
if b[i] < 0xff {
|
||||
b[i]++
|
||||
return string(b[:i+1])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func get(ctx context.Context, q queryer, k string) (string, bool, error) {
|
||||
var v string
|
||||
err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return v, true, nil
|
||||
}
|
||||
|
||||
// mgetBatch stays well under SQLite's default 32766 bound-parameter cap.
|
||||
const mgetBatch = 500
|
||||
|
||||
func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) {
|
||||
out := make(map[string]string, len(keys))
|
||||
for start := 0; start < len(keys); start += mgetBatch {
|
||||
end := min(start+mgetBatch, len(keys))
|
||||
chunk := keys[start:end]
|
||||
args := make([]any, len(chunk))
|
||||
for i, k := range chunk {
|
||||
args[i] = k
|
||||
}
|
||||
rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var k, v string
|
||||
if err := rows.Scan(&k, &v); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// rangeQuery returns keys with prefix, strictly after `after` (when non-empty),
|
||||
// in key order, at most limit (<= 0 means all).
|
||||
func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) {
|
||||
lo, loOp := prefix, ">="
|
||||
if after != "" && after >= prefix {
|
||||
lo, loOp = after, ">"
|
||||
}
|
||||
query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?`
|
||||
args := []any{lo}
|
||||
if end := PrefixEnd(prefix); end != "" {
|
||||
query += ` AND k < ?`
|
||||
args = append(args, end)
|
||||
}
|
||||
query += ` ORDER BY k`
|
||||
if limit > 0 {
|
||||
query += ` LIMIT ?`
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := q.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []KV
|
||||
for rows.Next() {
|
||||
var kv KV
|
||||
if err := rows.Scan(&kv.K, &kv.V); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, kv)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func count(ctx context.Context, q queryer, prefix string) (int64, error) {
|
||||
query := `SELECT count(*) FROM kv WHERE k >= ?`
|
||||
args := []any{prefix}
|
||||
if end := PrefixEnd(prefix); end != "" {
|
||||
query += ` AND k < ?`
|
||||
args = append(args, end)
|
||||
}
|
||||
var n int64
|
||||
err := q.QueryRowContext(ctx, query, args...).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// Get returns the committed value of k.
|
||||
func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) }
|
||||
|
||||
// MGet returns the committed values of the keys that exist.
|
||||
func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) {
|
||||
return mget(ctx, s.r, keys)
|
||||
}
|
||||
|
||||
// Range returns committed keys under prefix after `after`, in key order.
|
||||
func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
|
||||
return rangeQuery(ctx, s.r, prefix, after, limit)
|
||||
}
|
||||
|
||||
// Count returns how many committed keys sit under prefix.
|
||||
func (s *Store) Count(ctx context.Context, prefix string) (int64, error) {
|
||||
return count(ctx, s.r, prefix)
|
||||
}
|
||||
|
||||
// ScanPrefix walks every committed key under prefix in pages of `page`, each
|
||||
// page its own short read so a long walk never pins a WAL snapshot (which
|
||||
// would stop checkpoints and let the WAL grow without bound).
|
||||
func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error {
|
||||
if page <= 0 {
|
||||
page = 5000
|
||||
}
|
||||
after := ""
|
||||
for {
|
||||
batch, err := rangeQuery(ctx, s.r, prefix, after, page)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := fn(batch); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(batch) < page {
|
||||
return nil
|
||||
}
|
||||
after = batch[len(batch)-1].K
|
||||
}
|
||||
}
|
||||
|
||||
// CountAll counts every row. A full scan — callers cache it.
|
||||
func (s *Store) CountAll(ctx context.Context) (int64, error) {
|
||||
var n int64
|
||||
err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ErrClosed is returned by Update on a closed store.
|
||||
var ErrClosed = errors.New("kv: store closed")
|
||||
|
||||
// Update runs fn atomically: its writes (and the change events they publish)
|
||||
// commit together or not at all. fn may share a SQLite transaction with other
|
||||
// concurrent Updates (group commit) but is isolated from their failures by a
|
||||
// savepoint. Never call Update from inside fn.
|
||||
//
|
||||
// ctx bounds only the wait for the writer. Once fn starts it runs to
|
||||
// completion: interrupting a statement inside a shared transaction would roll
|
||||
// back every request in the group.
|
||||
func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error {
|
||||
req := &writeReq{fn: fn, done: make(chan error, 1)}
|
||||
select {
|
||||
case s.writes <- req:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-s.stopped:
|
||||
return ErrClosed
|
||||
}
|
||||
return <-req.done
|
||||
}
|
||||
|
||||
// writer is the single goroutine that executes writes.
|
||||
func (s *Store) writer() {
|
||||
for {
|
||||
var first *writeReq
|
||||
select {
|
||||
case first = <-s.writes:
|
||||
case <-s.stopped:
|
||||
return
|
||||
}
|
||||
group := []*writeReq{first}
|
||||
drain:
|
||||
for len(group) < maxGroup {
|
||||
select {
|
||||
case r := <-s.writes:
|
||||
group = append(group, r)
|
||||
default:
|
||||
break drain
|
||||
}
|
||||
}
|
||||
s.writeMu.Lock()
|
||||
s.runGroup(group)
|
||||
s.writeMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// group is the state shared by the requests of one commit.
|
||||
type group struct {
|
||||
tx *sql.Tx
|
||||
pending map[string]*streamPending
|
||||
stmts map[*sql.Stmt]*sql.Stmt
|
||||
}
|
||||
|
||||
func (g *group) snapshot() map[string]streamPending {
|
||||
out := make(map[string]streamPending, len(g.pending))
|
||||
for k, p := range g.pending {
|
||||
out[k] = *p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (g *group) restore(snap map[string]streamPending) {
|
||||
g.pending = make(map[string]*streamPending, len(snap))
|
||||
for k, p := range snap {
|
||||
p := p
|
||||
g.pending[k] = &p
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) runGroup(reqs []*writeReq) {
|
||||
ctx := context.Background()
|
||||
results := make([]error, len(reqs))
|
||||
fail := func(err error) {
|
||||
for i, r := range reqs {
|
||||
if results[i] == nil {
|
||||
results[i] = err
|
||||
}
|
||||
r.done <- results[i]
|
||||
}
|
||||
}
|
||||
|
||||
sqlTx, err := s.w.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
fail(fmt.Errorf("begin: %w", err))
|
||||
return
|
||||
}
|
||||
g := &group{tx: sqlTx}
|
||||
for i, r := range reqs {
|
||||
if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil {
|
||||
_ = sqlTx.Rollback()
|
||||
fail(fmt.Errorf("savepoint: %w", err))
|
||||
return
|
||||
}
|
||||
snap := g.snapshot()
|
||||
tx := &Tx{ctx: ctx, s: s, g: g}
|
||||
err := r.fn(tx)
|
||||
if err == nil {
|
||||
err = tx.flushEvents()
|
||||
}
|
||||
if err != nil {
|
||||
results[i] = err
|
||||
if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
|
||||
_ = sqlTx.Rollback()
|
||||
fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err))
|
||||
return
|
||||
}
|
||||
g.restore(snap)
|
||||
}
|
||||
if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil {
|
||||
_ = sqlTx.Rollback()
|
||||
fail(fmt.Errorf("release savepoint: %w", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
|
||||
_ = sqlTx.Rollback()
|
||||
fail(err)
|
||||
return
|
||||
}
|
||||
if err := sqlTx.Commit(); err != nil {
|
||||
fail(fmt.Errorf("commit: %w", err))
|
||||
return
|
||||
}
|
||||
s.afterCommit(g.pending)
|
||||
for i, r := range reqs {
|
||||
r.done <- results[i]
|
||||
}
|
||||
}
|
||||
|
||||
// Tx is one Update's view of the write transaction. Its reads see its own
|
||||
// writes (and those of earlier requests in the same group).
|
||||
type Tx struct {
|
||||
ctx context.Context
|
||||
s *Store
|
||||
g *group
|
||||
events []map[string]string
|
||||
}
|
||||
|
||||
func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
|
||||
|
||||
// stmt binds one of the store's prepared statements to the transaction.
|
||||
func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt {
|
||||
if t.g.stmts == nil {
|
||||
t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5)
|
||||
}
|
||||
bound := t.g.stmts[st]
|
||||
if bound == nil {
|
||||
bound = t.g.tx.StmtContext(t.ctx, st)
|
||||
t.g.stmts[st] = bound
|
||||
}
|
||||
return bound
|
||||
}
|
||||
|
||||
// Context is the context the transaction runs under.
|
||||
func (t *Tx) Context() context.Context { return t.ctx }
|
||||
|
||||
func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) {
|
||||
var v string
|
||||
err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return v, true, nil
|
||||
}
|
||||
func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) {
|
||||
return mget(ctx, t.sqlTx(), keys)
|
||||
}
|
||||
func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
|
||||
return rangeQuery(ctx, t.sqlTx(), prefix, after, limit)
|
||||
}
|
||||
func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) {
|
||||
return count(ctx, t.sqlTx(), prefix)
|
||||
}
|
||||
|
||||
// Exec runs a raw statement in the transaction — for the caller's derived
|
||||
// tables (FTS), never for kv rows (which would skip events).
|
||||
func (t *Tx) Exec(query string, args ...any) (sql.Result, error) {
|
||||
return t.sqlTx().ExecContext(t.ctx, query, args...)
|
||||
}
|
||||
|
||||
// QueryRow runs a raw single-row query in the transaction.
|
||||
func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
|
||||
return t.sqlTx().QueryRowContext(t.ctx, query, args...)
|
||||
}
|
||||
|
||||
// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
|
||||
// keyspace notifications did.
|
||||
func (t *Tx) Put(k, v string) error {
|
||||
if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
|
||||
return fmt.Errorf("put %q: %w", k, err)
|
||||
}
|
||||
t.event("set", k)
|
||||
return nil
|
||||
}
|
||||
|
||||
// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
|
||||
// Publishes nothing when the key already existed (nothing changed).
|
||||
func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
|
||||
res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("put %q: %w", k, err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n > 0 {
|
||||
t.event("set", k)
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// Delete removes k. Reports whether it existed; publishes `del` only then,
|
||||
// as Redis did.
|
||||
func (t *Tx) Delete(k string) (bool, error) {
|
||||
res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("delete %q: %w", k, err)
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
if n > 0 {
|
||||
t.event("del", k)
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
|
||||
// returns how many went. Bulk callers loop over separate Updates with a limit
|
||||
// so one transaction never grows to millions of rows.
|
||||
func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
|
||||
end := PrefixEnd(prefix)
|
||||
sel := `SELECT k FROM kv WHERE k >= ?`
|
||||
args := []any{prefix}
|
||||
if end != "" {
|
||||
sel += ` AND k < ?`
|
||||
args = append(args, end)
|
||||
}
|
||||
if limit > 0 {
|
||||
sel += ` ORDER BY k LIMIT ?`
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
|
||||
}
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, k := range keys {
|
||||
t.event("del", k)
|
||||
}
|
||||
return int64(len(keys)), nil
|
||||
}
|
||||
|
||||
func (t *Tx) event(op, k string) {
|
||||
if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) {
|
||||
return
|
||||
}
|
||||
t.events = append(t.events, map[string]string{
|
||||
"type": op,
|
||||
"key": k,
|
||||
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
|
||||
})
|
||||
}
|
||||
|
||||
// flushEvents appends the queued change events to the event stream, inside
|
||||
// the transaction, right before commit.
|
||||
func (t *Tx) flushEvents() error {
|
||||
if len(t.events) == 0 {
|
||||
return nil
|
||||
}
|
||||
evs := t.events
|
||||
t.events = nil
|
||||
for _, e := range evs {
|
||||
if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user