replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/storage/udl_admin.go

@@ -3,22 +3,16 @@ package storage
 // Account-scoped UDL administration: what a uid holds, and removing all of it.
 //
 // Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
-// cell — because there is no per-user cid index to consult. The Redis model
-// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
-// "what is in My List", not "everything this account ever wrote". Adding a
-// third index would put a write on every UDL upsert to serve two admin calls,
-// so these SCAN instead and accept being O(keyspace).
-//
-// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
-// one: the accounts page needs a number for every account, and MATCH filters
-// server-side without indexing, so N per-uid calls would walk the keyspace N
-// times. One walk buckets them all.
+// cell. Keys are ordered, so one account's cells are the key range under
+// `udl:rec:<uid>/`, and every account's are the range under `udl:rec:`.
+// UDLAllUserStats stays a bulk call (one walk buckets every account) because
+// the accounts page needs a number for each.
 
 import (
-	"context"
 	"fmt"
 	"strings"
-	"time"
+
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 // UDLUserStats counts what one account holds. Records is the number of cells,
@@ -30,11 +24,8 @@ type UDLUserStats struct {
 	Keys    int `json:"keys"`
 }
 
-// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
-const udlScanBatch = 1000
-
-// safeUIDForPattern rejects anything that would change the meaning of a SCAN
-// glob or reach across the key structure. meta-core's uids are alphanumeric
+// safeUIDForPattern rejects anything that would reach across the key
+// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric
 // (identity.validUID), so this only ever fires on a hostile or malformed one —
 // but it fires here, in the layer that builds the pattern, rather than relying
 // on every caller having validated first.
@@ -62,118 +53,76 @@ func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
 	return parts[0], parts[1], parts[2], true
 }
 
-// scanKeys walks the keyspace for pattern and returns every match. Caller holds
-// the lock.
-//
-// SCAN can return the same key more than once across cursor iterations (it
-// guarantees no misses, not no duplicates), so callers that count must
-// de-duplicate — both callers below collect into maps or sets for that reason.
-func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
-	var (
-		out    []string
-		cursor uint64
-	)
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan %q: %w", pattern, err)
-		}
-		out = append(out, keys...)
-		cursor = next
-		if cursor == 0 {
-			return out, nil
-		}
-	}
-}
-
-// UDLUserCells returns every (cid, key) pair this account holds a cell for.
-// De-duplicated, since SCAN may repeat a key.
+// UDLUserCells returns every (cid, key) pair this account holds a cell for,
+// keyed by the "cid/key" tail with the cid as value (the purge needs them
+// grouped).
 func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
-	// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
-	// the cid, which the purge needs grouped.
 	if err := safeUIDForPattern(uid); err != nil {
 		return nil, err
 	}
-
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	prefix := c.buildKey("udl:rec:")
-	keys, err := c.scanKeys(ctx, prefix+uid+"/*")
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
-	out := make(map[string]string, len(keys))
-	for _, full := range keys {
-		gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
-		// The glob can only match this uid, but a uid that is a prefix of
-		// another would still be caught by the '/' in the pattern; check anyway
-		// so a malformed key can never be attributed to the wrong account.
-		if !ok || gotUID != uid {
-			continue
+	ctx, cancel := opCtx()
+	defer cancel()
+
+	out := make(map[string]string)
+	err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error {
+		for _, row := range rows {
+			gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
+			if !ok || gotUID != uid {
+				continue
+			}
+			out[cid+"/"+key] = cid
 		}
-		out[cid+"/"+key] = cid
-	}
-	return out, nil
+		return nil
+	})
+	return out, err
 }
 
-// UDLAllUserStats counts every account's cells in one keyspace walk.
+// UDLAllUserStats counts every account's cells in one walk.
 //
 // Returns only accounts that actually hold records; a uid absent from the map
 // holds none. Callers render a zero for those rather than treating it as an
 // error — an account created a minute ago legitimately has nothing.
 func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-
-	prefix := c.buildKey("udl:rec:")
-	keys, err := c.scanKeys(ctx, prefix+"*")
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
+	ctx, cancel := opCtx()
+	defer cancel()
 
-	// Sets rather than counters: SCAN may hand back the same key twice, which
-	// would otherwise inflate every number on the page.
 	type acc struct {
-		cells map[string]struct{}
+		cells int
 		cids  map[string]struct{}
 		keys  map[string]struct{}
 	}
 	byUID := make(map[string]*acc)
-	for _, full := range keys {
-		uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
-		if !ok {
-			continue
-		}
-		a := byUID[uid]
-		if a == nil {
-			a = &acc{
-				cells: make(map[string]struct{}),
-				cids:  make(map[string]struct{}),
-				keys:  make(map[string]struct{}),
+	err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error {
+		for _, row := range rows {
+			uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
+			if !ok {
+				continue
 			}
-			byUID[uid] = a
+			a := byUID[uid]
+			if a == nil {
+				a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})}
+				byUID[uid] = a
+			}
+			a.cells++
+			a.cids[cid] = struct{}{}
+			a.keys[key] = struct{}{}
 		}
-		a.cells[cid+"/"+key] = struct{}{}
-		a.cids[cid] = struct{}{}
-		a.keys[key] = struct{}{}
+		return nil
+	})
+	if err != nil {
+		return nil, err
 	}
-
 	out := make(map[string]UDLUserStats, len(byUID))
 	for uid, a := range byUID {
-		out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
+		out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)}
 	}
 	return out, nil
 }
@@ -185,63 +134,46 @@ func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
 // The shared indexes are the part that is easy to miss and expensive to leave.
 // `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
 // not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
-// uid listed as an opinion-holder on every title it ever touched. Aggregation
-// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
-// rating average stays right, but uids_for_cid keeps naming a deleted account
-// forever and every aggregate read pays for the lookup.
+// uid listed as an opinion-holder on every title it ever touched.
 //
 // Idempotent: purging an account with nothing stored is a no-op returning
-// zeroes, so a retry after a partial failure is safe.
+// zeroes, so a retry is safe.
 func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
 	cells, err := c.UDLUserCells(uid)
 	if err != nil {
 		return UDLUserStats{}, err
 	}
 
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.client == nil {
-		return UDLUserStats{}, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-
 	stats := UDLUserStats{Records: len(cells)}
 	distinctCids := make(map[string]struct{})
 	distinctKeys := make(map[string]struct{})
-
-	pipe := c.client.Pipeline()
-	for cidKey, cid := range cells {
-		key := strings.TrimPrefix(cidKey, cid+"/")
-		distinctCids[cid] = struct{}{}
-		distinctKeys[key] = struct{}{}
-		pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
-		pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
-	}
-	for cid := range distinctCids {
-		pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
+	err = c.update(func(tx *kv.Tx) error {
+		for cidKey, cid := range cells {
+			key := strings.TrimPrefix(cidKey, cid+"/")
+			distinctCids[cid] = struct{}{}
+			distinctKeys[key] = struct{}{}
+			if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil {
+				return err
+			}
+			if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil {
+				return err
+			}
+		}
+		for cid := range distinctCids {
+			if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil {
+				return err
+			}
+		}
+		// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one
+		// key range, dropped whole — which also catches index entries whose
+		// cell is already gone (the state a half-finished purge leaves).
+		_, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0)
+		return err
+	})
+	if err != nil {
+		return UDLUserStats{}, fmt.Errorf("purge: %w", err)
 	}
 	stats.Cids = len(distinctCids)
 	stats.Keys = len(distinctKeys)
-
-	// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are enumerable
-	// directly, and are dropped whole rather than member-by-member. The glob is
-	// what keeps this correct as indexes are added — `:cids` needed no change
-	// here when it landed. Scanning them separately also catches
-	// index entries whose cell is already gone — exactly the state a previously
-	// half-finished purge leaves behind.
-	idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
-	idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
-	if err != nil {
-		return UDLUserStats{}, err
-	}
-	for _, k := range idxKeys {
-		pipe.Del(ctx, k)
-	}
-
-	if _, err := pipe.Exec(ctx); err != nil {
-		return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
-	}
 	return stats, nil
 }