v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410

internal/storage/client.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"log"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -16,21 +15,31 @@ import (
 	"github.com/metazla/meta-core/internal/schema"
 )
 
-// Key layout (everything is one string→string table, see internal/kv):
+// meta-core keeps two SQLite files (docs/implementation/metacore-claims-model.md):
 //
-//	file:<rid>/<field>       a record property — the ONLY thing under `file:`
-//	cid:<cid>                reverse index → <rid>
-//	idx:roots/<rid>          the set of every root (was the Redis SET file:__index__)
-//	idx:fts/<rid>            rowid of the record's row in the FTS table
-//	idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
-//	dup:<rid>/<path>         a duplicate path of the record (was the SET file:<rid>/duplicates)
-//	udl:…                    User Data Layer, see udl.go
-//	ev:<stream>/…, evhwm:…   event streams, see internal/kv/stream.go
+//   - the TRUTH database (truth.go): principal, claim, cid_group, meta — the
+//     signed votes and the locally minted rid names, primary keys only, backed
+//     up;
+//
+//   - the CACHE database: everything derived from truth, never backed up,
+//     rebuilt when missing (apply.go). Its kv table keeps the v1 layout:
+//
+//     file:<rid>/<field>       the MERGED record view — the ONLY thing under `file:`
+//     cid:<cid>                cid → rid lookup (copy of truth's cid_group)
+//     idx:roots/<rid>          the set of every record
+//     idx:fts/<rid>            rowid of the record's row in the FTS table
+//     idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
+//     ev:<stream>/…, evhwm:…   event streams, see internal/kv/stream.go
+//     meta:applied_seq, meta:rebuilt_at   apply-loop bookkeeping
+//     anything else            service scratch KV (btmagnet:*, searchcov:*), /api/kv/value
+//
+//     plus the tables claim_by_pid and local_file (localfile.go), and the
+//     field index + FTS (search.go).
 //
 // Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is
-// exactly the records, so a record is the key range [file:<rid>/, file:<rid>0),
-// the field expression index covers only real fields, and every change under
-// `file:` is a real field change worth publishing on meta:events.
+// exactly the merged records, so a record is the key range [file:<rid>/,
+// file:<rid>0), the field expression index covers only real fields, and every
+// change under `file:` is a real field change worth publishing on meta:events.
 const (
 	recordNS   = "file:"
 	rootsNS    = "idx:roots/"
@@ -43,10 +52,9 @@ const (
 	readerPage = 5_000
 )
 
-// MetaEventsStream receives a set/del entry for every record-key change. It is
-// written by the storage engine in the same transaction as the change (the
-// replacement for Redis keyspace notifications), and served as SSE by
-// /api/events/meta.
+// MetaEventsStream receives a set/del entry for every merged-record change. It
+// is written by the cache engine in the same transaction as the change, and
+// served as SSE by /api/events/meta.
 const MetaEventsStream = "meta:events"
 
 // MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is
@@ -54,10 +62,13 @@ const MetaEventsStream = "meta:events"
 // "Backing-store retention and lifecycle".
 const MetaEventsMaxLen = 100_000
 
-// Client is meta-core's metadata store.
+// Client is meta-core's metadata store: the truth database, the cache, and
+// the apply loop between them.
 type Client struct {
-	store *kv.Store
-	mu    sync.RWMutex
+	store   *kv.Store // cache
+	truth   *kv.Store
+	applier *applier
+	mu      sync.RWMutex
 
 	keysMu    sync.Mutex
 	keysCount int64
@@ -69,60 +80,98 @@ func NewClient() *Client {
 	return &Client{}
 }
 
-// Connect opens (creating if needed) the database file at path.
-func (c *Client) Connect(path string) error {
+// Connect opens (creating if needed) the truth database at truthPath and the
+// cache at cachePath. Call EnsureSchemaVersion, then Start.
+func (c *Client) Connect(truthPath, cachePath string) error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	store, err := kv.Open(path, kv.Options{
+	truth, err := kv.Open(truthPath, kv.Options{
+		NoKVTable:     true,
+		Schema:        truthSchema,
+		WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
+		ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
+		Readers:       envInt("META_CORE_DB_READERS", 8),
+	})
+	if err != nil {
+		return fmt.Errorf("open truth %s: %w", truthPath, err)
+	}
+	store, err := kv.Open(cachePath, kv.Options{
 		EventStream:   MetaEventsStream,
 		EventMaxLen:   MetaEventsMaxLen,
 		EventKey:      func(k string) bool { return strings.HasPrefix(k, recordNS) },
 		WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
 		ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
 		Readers:       envInt("META_CORE_DB_READERS", 8),
-		Schema:        searchSchema,
+		Schema:        searchSchema + cacheTablesSchema,
 		OnChange:      onChange,
+		Synchronous:   "OFF",
 	})
 	if err != nil {
-		return fmt.Errorf("open %s: %w", path, err)
+		truth.Close()
+		return fmt.Errorf("open cache %s: %w", cachePath, err)
 	}
-	c.store = store
-	log.Printf("[Storage] Opened %s", path)
+	c.truth, c.store = truth, store
+	log.Printf("[Storage] Opened truth %s, cache %s", truthPath, cachePath)
 	return nil
 }
 
-// Close closes the database.
+// Start brings the cache level with truth (replay or rebuild) and starts the
+// apply loop. Writes before Start fail.
+func (c *Client) Start() error {
+	applied, err := c.catchUp()
+	if err != nil {
+		return fmt.Errorf("cache catch-up: %w", err)
+	}
+	c.mu.Lock()
+	c.applier = newApplier(c, applied)
+	c.mu.Unlock()
+	return nil
+}
+
+// Close stops the apply loop and closes both databases.
 func (c *Client) Close() error {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if c.store == nil {
-		return nil
+	if c.applier != nil {
+		c.applier.close()
+		c.applier = nil
 	}
-	err := c.store.Close()
-	c.store = nil
-	return err
+	var errs []error
+	if c.store != nil {
+		errs = append(errs, c.store.Close())
+		c.store = nil
+	}
+	if c.truth != nil {
+		errs = append(errs, c.truth.Close())
+		c.truth = nil
+	}
+	return errors.Join(errs...)
 }
 
-// IsConnected reports whether the database is open.
+// IsConnected reports whether the databases are open.
 func (c *Client) IsConnected() bool {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
-	return c.store != nil
+	return c.store != nil && c.truth != nil
 }
 
-// Health checks that the database answers.
+// Health checks that both databases answer.
 func (c *Client) Health() bool {
 	s, err := c.kv()
 	if err != nil {
 		return false
 	}
+	t, err := c.truthStore()
+	if err != nil {
+		return false
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
-	return s.Ping(ctx) == nil
+	return s.Ping(ctx) == nil && t.Ping(ctx) == nil
 }
 
-// Store exposes the underlying engine (event streams for SSE and the schema
+// Store exposes the cache engine (event streams for SSE and the schema
 // indexer).
 func (c *Client) Store() *kv.Store {
 	c.mu.RLock()
@@ -139,7 +188,16 @@ func (c *Client) kv() (*kv.Store, error) {
 	return c.store, nil
 }
 
-// update runs fn in one write transaction with the standard timeout.
+func (c *Client) truthStore() (*kv.Store, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.truth == nil || c.applier == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+	return c.truth, nil
+}
+
+// update runs fn in one cache write transaction with the standard timeout.
 func (c *Client) update(fn func(tx *kv.Tx) error) error {
 	s, err := c.kv()
 	if err != nil {
@@ -184,8 +242,92 @@ func splitRecordKey(key string) (hashID, field string, ok bool) {
 // isInternalKey reports keys that are derived bookkeeping, which the raw KV
 // editor must never write (it would desynchronise the derived state).
 func isInternalKey(key string) bool {
-	return strings.HasPrefix(key, "idx:") || strings.HasPrefix(key, dupNS) ||
-		strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS)
+	for _, p := range []string{recordNS, CIDIndexPrefix, "idx:", dupNS, streamNS, hwmNS, "meta:", "meta-core:"} {
+		if strings.HasPrefix(key, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// SetServiceValue writes service scratch KV (btmagnet:*, searchcov:*) — the
+// only raw key writes left. Merged records and bookkeeping are refused.
+func (c *Client) SetServiceValue(key, value string) error {
+	if key == "" || isInternalKey(key) {
+		return fmt.Errorf("key %q is not a service key", key)
+	}
+	return c.update(func(tx *kv.Tx) error { return tx.Put(key, value) })
+}
+
+// DeleteServiceValue deletes a service scratch key. Returns true if it existed.
+func (c *Client) DeleteServiceValue(key string) (bool, error) {
+	if key == "" || isInternalKey(key) {
+		return false, fmt.Errorf("key %q is not a service key", key)
+	}
+	existed := false
+	err := c.update(func(tx *kv.Tx) error {
+		var err error
+		existed, err = tx.Delete(key)
+		return err
+	})
+	return existed, err
+}
+
+// ClearAllMetadata wipes BOTH databases: every claim, signer and group in
+// truth, and every derived row of the cache. Irreversible — truth holds votes
+// nothing else can recover. Returns the number of records that existed.
+func (c *Client) ClearAllMetadata() (int64, error) {
+	truth, err := c.truthStore()
+	if err != nil {
+		return 0, err
+	}
+	roots, err := c.CountRoots()
+	if err != nil {
+		return 0, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	if err := truth.Update(ctx, func(tx *kv.Tx) error {
+		for _, t := range []string{"claim", "principal", "cid_group"} {
+			if _, err := tx.Exec(`DELETE FROM ` + t); err != nil {
+				return err
+			}
+		}
+		_, err := tx.Exec(`DELETE FROM meta WHERE name = ?`, lastSeqName)
+		return err
+	}); err != nil {
+		return 0, fmt.Errorf("clear truth: %w", err)
+	}
+	for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
+		for {
+			var n int64
+			if err := c.update(func(tx *kv.Tx) error {
+				var err error
+				n, err = tx.DeletePrefix(prefix, bulkBatch)
+				return err
+			}); err != nil {
+				return 0, fmt.Errorf("clear %s: %w", prefix, err)
+			}
+			if n < bulkBatch {
+				break
+			}
+		}
+	}
+	if err := c.update(func(tx *kv.Tx) error {
+		for _, q := range []string{`DELETE FROM fts`, `DELETE FROM claim_by_pid`, `DELETE FROM local_file`} {
+			if _, err := tx.Exec(q); err != nil {
+				return err
+			}
+		}
+		return tx.Put(appliedSeqKey, "0")
+	}); err != nil {
+		return 0, fmt.Errorf("clear cache: %w", err)
+	}
+	c.applier.mu.Lock()
+	c.applier.applied = 0
+	c.applier.mu.Unlock()
+	log.Printf("[Storage] Cleared both databases (%d records)", roots)
+	return roots, nil
 }
 
 // Derived counters, maintained by onChange in the same transaction as the
@@ -300,44 +442,6 @@ func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]str
 	return out, nil
 }
 
-// writeFieldsTx is the one write path for record fields: the keys, the root
-// index, the CID reverse aliases and the search row, all in the caller's
-// transaction.
-func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
-	if hashID == "" {
-		return fmt.Errorf("empty hash id")
-	}
-	prefix := recordPrefix(hashID)
-	searchChanged := false
-	for field, value := range metadata {
-		if isReservedFieldName(field) {
-			continue
-		}
-		changed, err := tx.Set(prefix+field, value)
-		if err != nil {
-			return err
-		}
-		searchChanged = searchChanged || (changed && affectsHaystack(field))
-	}
-	newRoot, err := tx.Set(rootKey(hashID), "")
-	if err != nil {
-		return err
-	}
-	for field := range metadata {
-		if cidStr := cidFromKeysetField(field); cidStr != "" {
-			if err := addAliasTx(tx, hashID, cidStr); err != nil {
-				return err
-			}
-		}
-	}
-	// The FTS row is rewritten only when the haystack can have changed — a
-	// re-persisted record whose values are all in place costs reads only.
-	if searchChanged || newRoot {
-		return refreshSearchTx(tx, hashID)
-	}
-	return nil
-}
-
 // GetMetadataFlat retrieves all metadata for a file as a flat map.
 func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	s, err := c.kv()
@@ -349,28 +453,6 @@ func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
 	return readRecord(ctx, s, hashID)
 }
 
-// SetMetadataFlat stores metadata for a file using flat keys (merge: fields
-// absent from metadata are kept — the Redis implementation never cleared
-// either).
-func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error {
-	if len(metadata) == 0 {
-		return nil
-	}
-	return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) })
-}
-
-// MergeMetadataFlat merges new metadata into existing (PATCH semantics): new
-// keys are added, existing keys are updated, missing keys are NOT deleted.
-func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) {
-	if len(metadata) == 0 {
-		return 0, nil
-	}
-	if err := c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }); err != nil {
-		return 0, err
-	}
-	return len(metadata), nil
-}
-
 // GetAllHashIDs returns every root id, sorted.
 func (c *Client) GetAllHashIDs() ([]string, error) {
 	s, err := c.kv()
@@ -401,57 +483,6 @@ func (c *Client) GetProperty(hashID, property string) (string, error) {
 	return v, err
 }
 
-// SetProperty sets a single property value.
-func (c *Client) SetProperty(hashID, property, value string) error {
-	if isReservedFieldName(property) {
-		return fmt.Errorf("field name %q is reserved", property)
-	}
-	return c.update(func(tx *kv.Tx) error {
-		return writeFieldsTx(tx, hashID, map[string]string{property: value})
-	})
-}
-
-// deleteRecordTx removes everything keyed off hashID: its properties, the
-// reverse-index entries its cids/ members own, its duplicates, its root-index
-// entry and its search row. Returns the number of property keys removed.
-func deleteRecordTx(tx *kv.Tx, hashID string) (int64, error) {
-	ctx := tx.Context()
-	cids, err := cidsForRoot(ctx, tx, hashID)
-	if err != nil {
-		return 0, err
-	}
-	for _, cidStr := range cids {
-		if _, err := tx.Delete(CIDIndexPrefix + cidStr); err != nil {
-			return 0, err
-		}
-	}
-	n, err := tx.DeletePrefix(recordPrefix(hashID), 0)
-	if err != nil {
-		return 0, err
-	}
-	if _, err := tx.DeletePrefix(dupPrefix(hashID), 0); err != nil {
-		return n, err
-	}
-	if _, err := tx.Delete(rootKey(hashID)); err != nil {
-		return n, err
-	}
-	return n, deleteSearchRowTx(tx, hashID)
-}
-
-// DeleteMetadata deletes all metadata for a file, and unmaps every
-// reverse-index entry registered for this root, so a subsequent
-// GET /api/meta/<old-cid> correctly returns 404 instead of pointing at a root
-// with no keys.
-func (c *Client) DeleteMetadata(hashID string) (int64, error) {
-	var n int64
-	err := c.update(func(tx *kv.Tx) error {
-		var err error
-		n, err = deleteRecordTx(tx, hashID)
-		return err
-	})
-	return n, err
-}
-
 // CountFiles returns the number of indexed roots.
 func (c *Client) CountFiles() (int, error) {
 	n, err := c.CountRoots()
@@ -496,161 +527,28 @@ func normalizeFilesRelativePath(p string) string {
 
 // DeleteProperty deletes a single property.
 //
-// For a cids/<cid> member this is the full inverse of the write: the flat key
-// AND the cid:<cid> reverse alias both go. Anything less leaves the CID still
-// resolving to this record — see removeAliasTx.
-func (c *Client) DeleteProperty(hashID, property string) error {
-	return c.update(func(tx *kv.Tx) error {
-		if _, err := tx.Delete(recordPrefix(hashID) + property); err != nil {
-			return err
-		}
-		if cidStr := cidFromKeysetField(property); cidStr != "" {
-			if err := removeAliasTx(tx, hashID, cidStr); err != nil {
-				return fmt.Errorf("remove alias for %s: %w", cidStr, err)
-			}
-		}
-		if affectsHaystack(property) {
-			return refreshSearchTx(tx, hashID)
-		}
-		return nil
-	})
-}
-
-// AddToSet adds a value to a set-type field (stored as a pipe-delimited string
-// in one flat key). Returns true if value was added, false if it already
-// existed.
-func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
-	added := false
-	err := c.update(func(tx *kv.Tx) error {
-		current, _, err := tx.Get(tx.Context(), recordPrefix(hashID)+property)
-		if err != nil {
-			return err
-		}
-		var values []string
-		if current != "" {
-			values = strings.Split(current, "|")
-		}
-		for _, v := range values {
-			if v == value {
-				return nil
-			}
-		}
-		added = true
-		return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(append(values, value), "|")})
-	})
-	return added, err
-}
-
-// RemoveFromSet removes a value from a set-type field (stored in one flat key).
-// Returns true if value was removed, false if it didn't exist.
-func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
-	removed := false
-	err := c.update(func(tx *kv.Tx) error {
-		key := recordPrefix(hashID) + property
-		current, ok, err := tx.Get(tx.Context(), key)
-		if err != nil || !ok {
-			return err
-		}
-		values := strings.Split(current, "|")
-		kept := make([]string, 0, len(values))
-		for _, v := range values {
-			if v == value {
-				removed = true
-			} else {
-				kept = append(kept, v)
-			}
-		}
-		if !removed {
-			return nil
-		}
-		if len(kept) == 0 {
-			if _, err := tx.Delete(key); err != nil {
-				return err
-			}
-			if affectsHaystack(property) {
-				return refreshSearchTx(tx, hashID)
-			}
-			return nil
-		}
-		return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(kept, "|")})
-	})
-	return removed, err
-}
-
-// FileTuple holds the (uuid, path, size, mtime) record for one file. Used by
-// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple
-// matches what's already stored.
-type FileTuple struct {
-	HashID    string
-	FilePath  string
-	Size      int64
-	MtimeNano int64
-}
-
-// GetTuplesForAllFiles returns one FileTuple per indexed root, in batched
-// multi-key reads. Skips files missing any of filePath, sizeByte, mtimeNano.
-func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
-	s, err := c.kv()
-	if err != nil {
-		return nil, err
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
-	defer cancel()
-
-	fields := [3]string{"filePath", "sizeByte", "mtimeNano"}
-	var out []FileTuple
-	err = s.ScanPrefix(ctx, rootsNS, 1000, func(rows []kv.KV) error {
-		keys := make([]string, 0, len(rows)*len(fields))
-		for _, row := range rows {
-			prefix := recordPrefix(row.K[len(rootsNS):])
-			for _, f := range fields {
-				keys = append(keys, prefix+f)
-			}
-		}
-		vals, err := s.MGet(ctx, keys)
-		if err != nil {
-			return err
-		}
-		for _, row := range rows {
-			h := row.K[len(rootsNS):]
-			prefix := recordPrefix(h)
-			filePath, sizeStr, mtimeStr := vals[prefix+fields[0]], vals[prefix+fields[1]], vals[prefix+fields[2]]
-			if filePath == "" || sizeStr == "" || mtimeStr == "" {
-				continue
-			}
-			size, err := strconv.ParseInt(sizeStr, 10, 64)
-			if err != nil {
-				continue
-			}
-			mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
-			if err != nil {
-				continue
-			}
-			out = append(out, FileTuple{HashID: h, FilePath: filePath, Size: size, MtimeNano: mtime})
-		}
-		return nil
-	})
-	return out, err
-}
 
 // GetMemoryInfo reports the store's footprint for dashboards: the database
 // file plus its WAL, human-readable. (Under Redis this was used_memory_human;
 // the data no longer lives in RAM, so the disk footprint is the honest figure.)
 func (c *Client) GetMemoryInfo() (string, error) {
-	s, err := c.kv()
-	if err != nil {
+	if _, err := c.kv(); err != nil {
 		return "", err
 	}
-	return humanBytes(s.FileSize()), nil
+	return humanBytes(c.DBSize()), nil
 }
 
-// DBSize is the database file plus WAL, in bytes.
+// DBSize is both database files plus their WALs, in bytes.
 func (c *Client) DBSize() int64 {
-	s, err := c.kv()
-	if err != nil {
-		return 0
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	var n int64
+	for _, st := range []*kv.Store{c.store, c.truth} {
+		if st != nil {
+			n += st.FileSize()
+		}
 	}
-	return s.FileSize()
+	return n
 }
 
 func humanBytes(n int64) string {
@@ -674,37 +572,6 @@ func humanBytes(n int64) string {
 // redirects the next /meta/<cid> write back into that dead root, resurrecting
 // a zombie record under an old identity.
 //
-// Runs as many bounded transactions rather than one: a single transaction over
-// millions of rows would hold the writer for minutes and balloon the WAL.
-func (c *Client) ClearAllMetadata() (int64, error) {
-	roots, err := c.CountRoots()
-	if err != nil {
-		return 0, err
-	}
-	for _, prefix := range []string{recordNS, CIDIndexPrefix, dupNS, rootsNS, ftsNS} {
-		for {
-			var n int64
-			if err := c.update(func(tx *kv.Tx) error {
-				var err error
-				n, err = tx.DeletePrefix(prefix, bulkBatch)
-				return err
-			}); err != nil {
-				return 0, fmt.Errorf("clear %s: %w", prefix, err)
-			}
-			if n < bulkBatch {
-				break
-			}
-		}
-	}
-	if err := c.update(func(tx *kv.Tx) error {
-		_, err := tx.Exec(`DELETE FROM fts`)
-		return err
-	}); err != nil {
-		return 0, fmt.Errorf("clear search rows: %w", err)
-	}
-	log.Printf("[Storage] Cleared %d file metadata entries", roots)
-	return roots, nil
-}
 
 // ScanKeys lists the immediate children at the given prefix, S3-style.
 //
@@ -767,11 +634,24 @@ func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) (
 		leaves = append(leaves, k)
 		from, op = k, ">"
 	}
+	leaves, branches = serviceOnly(leaves), serviceOnly(branches)
 	sort.Strings(leaves)
 	sort.Strings(branches)
 	return leaves, branches, next, nil
 }
 
+// serviceOnly drops merged-record and bookkeeping keys: the /api/kv browser
+// shows service scratch KV only.
+func serviceOnly(keys []string) []string {
+	out := keys[:0]
+	for _, k := range keys {
+		if !isInternalKey(k) {
+			out = append(out, k)
+		}
+	}
+	return out
+}
+
 // FindMatch is a single value-search hit.
 type FindMatch struct {
 	Key       string `json:"key"`
@@ -876,10 +756,11 @@ func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error)
 	if err := rows.Err(); err != nil {
 		return nil, false, err
 	}
-	if len(results) > limit {
-		return results[:limit], true, nil
+	truncated := len(results) > limit
+	if truncated {
+		results = results[:limit]
 	}
-	return results, false, nil
+	return serviceOnly(results), truncated, nil
 }
 
 // GetKeyInfo returns the key's type — "string" when it exists (every row is a
@@ -889,6 +770,9 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool,
 	if err != nil {
 		return "", "", false, err
 	}
+	if isInternalKey(key) {
+		return "", "", false, nil
+	}
 	ctx, cancel := opCtx()
 	defer cancel()
 	v, ok, err := s.Get(ctx, key)
@@ -898,70 +782,6 @@ func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool,
 	return "string", v, true, nil
 }
 
-// SetRawValue writes a string value at the exact key (the raw KV editor).
-// Refuses derived bookkeeping keys. A record key keeps the search row honest
-// like every other write path.
-func (c *Client) SetRawValue(key, value string) error {
-	if isInternalKey(key) {
-		return fmt.Errorf("key %q is internal bookkeeping and not editable", key)
-	}
-	return c.update(func(tx *kv.Tx) error {
-		if err := tx.Put(key, value); err != nil {
-			return err
-		}
-		if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
-			return refreshSearchTx(tx, hashID)
-		}
-		return nil
-	})
-}
-
-// DeleteRawKey deletes the exact key. Returns true if the key existed.
-func (c *Client) DeleteRawKey(key string) (bool, error) {
-	if isInternalKey(key) {
-		return false, fmt.Errorf("key %q is internal bookkeeping and not editable", key)
-	}
-	existed := false
-	err := c.update(func(tx *kv.Tx) error {
-		var err error
-		if existed, err = tx.Delete(key); err != nil || !existed {
-			return err
-		}
-		if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
-			return refreshSearchTx(tx, hashID)
-		}
-		return nil
-	})
-	return existed, err
-}
-
-// IndexAdd adds a hash ID to the root index. Idempotent.
-func (c *Client) IndexAdd(hashID string) error {
-	return c.update(func(tx *kv.Tx) error {
-		if err := tx.Put(rootKey(hashID), ""); err != nil {
-			return err
-		}
-		return ensureSearchRowTx(tx, hashID)
-	})
-}
-
-// IndexRemoveIfEmpty removes hashID from the root index (and drops its search
-// row) when no file:<hashID>/* keys remain. Returns true if it was removed.
-func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
-	removed := false
-	err := c.update(func(tx *kv.Tx) error {
-		n, err := tx.Count(tx.Context(), recordPrefix(hashID))
-		if err != nil || n > 0 {
-			return err
-		}
-		if removed, err = tx.Delete(rootKey(hashID)); err != nil {
-			return err
-		}
-		return deleteSearchRowTx(tx, hashID)
-	})
-	return removed, err
-}
-
 // GetPrefix returns the key namespace prefix. Always empty: meta-core owns its
 // database file outright, so there is nothing to share a keyspace with.
 func (c *Client) GetPrefix() string { return "" }