meta-core/internal/storage/client.go

package storage

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"log"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/metazla/meta-core/internal/kv"
	"github.com/metazla/meta-core/internal/schema"
)

// Key layout (everything is one string→string table, see internal/kv):
//
//	file:<rid>/<field>       a record property — the ONLY thing under `file:`
//	cid:<cid>                reverse index → <rid>
//	idx:roots/<rid>          the set of every root (was the Redis SET file:__index__)
//	idx:fts/<rid>            rowid of the record's row in the FTS table
//	idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
//	dup:<rid>/<path>         a duplicate path of the record (was the SET file:<rid>/duplicates)
//	udl:…                    User Data Layer, see udl.go
//	ev:<stream>/…, evhwm:…   event streams, see internal/kv/stream.go
//
// Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is
// exactly the records, so a record is the key range [file:<rid>/, file:<rid>0),
// the field expression index covers only real fields, and every change under
// `file:` is a real field change worth publishing on meta:events.
const (
	recordNS   = "file:"
	rootsNS    = "idx:roots/"
	ftsNS      = "idx:fts/"
	dupNS      = "dup:"
	streamNS   = "ev:"
	hwmNS      = "evhwm:"
	opTimeout  = 30 * time.Second
	bulkBatch  = 10_000
	readerPage = 5_000
)

// MetaEventsStream receives a set/del entry for every record-key change. It is
// written by the storage engine in the same transaction as the change (the
// replacement for Redis keyspace notifications), and served as SSE by
// /api/events/meta.
const MetaEventsStream = "meta:events"

// MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is
// a contract requirement for the SSE wire — see docs/api-mediated-access.md
// "Backing-store retention and lifecycle".
const MetaEventsMaxLen = 100_000

// Client is meta-core's metadata store.
type Client struct {
	store *kv.Store
	mu    sync.RWMutex

	keysMu    sync.Mutex
	keysCount int64
	keysAt    time.Time
}

// NewClient creates an unconnected client.
func NewClient() *Client {
	return &Client{}
}

// Connect opens (creating if needed) the database file at path.
func (c *Client) Connect(path string) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	store, err := kv.Open(path, kv.Options{
		EventStream:   MetaEventsStream,
		EventMaxLen:   MetaEventsMaxLen,
		EventKey:      func(k string) bool { return strings.HasPrefix(k, recordNS) },
		WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
		ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
		Readers:       envInt("META_CORE_DB_READERS", 8),
		Schema:        searchSchema,
		OnChange:      onChange,
	})
	if err != nil {
		return fmt.Errorf("open %s: %w", path, err)
	}
	c.store = store
	log.Printf("[Storage] Opened %s", path)
	return nil
}

// Close closes the database.
func (c *Client) Close() error {
	c.mu.Lock()
	defer c.mu.Unlock()
	if c.store == nil {
		return nil
	}
	err := c.store.Close()
	c.store = nil
	return err
}

// IsConnected reports whether the database is open.
func (c *Client) IsConnected() bool {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.store != nil
}

// Health checks that the database answers.
func (c *Client) Health() bool {
	s, err := c.kv()
	if err != nil {
		return false
	}
	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	defer cancel()
	return s.Ping(ctx) == nil
}

// Store exposes the underlying engine (event streams for SSE and the schema
// indexer).
func (c *Client) Store() *kv.Store {
	c.mu.RLock()
	defer c.mu.RUnlock()
	return c.store
}

func (c *Client) kv() (*kv.Store, error) {
	c.mu.RLock()
	defer c.mu.RUnlock()
	if c.store == nil {
		return nil, fmt.Errorf("not connected")
	}
	return c.store, nil
}

// update runs fn in one write transaction with the standard timeout.
func (c *Client) update(fn func(tx *kv.Tx) error) error {
	s, err := c.kv()
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), opTimeout)
	defer cancel()
	return s.Update(ctx, fn)
}

func opCtx() (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.Background(), opTimeout)
}

func recordPrefix(hashID string) string { return recordNS + hashID + "/" }
func rootKey(hashID string) string      { return rootsNS + hashID }
func dupPrefix(hashID string) string    { return dupNS + hashID + "/" }

// isReservedFieldName reports whether field names bookkeeping rather than
// metadata. `__fields__` was the Redis-era per-record field index; it no
// longer exists, but a payload carrying it (an old snapshot) is still refused
// so it can never become a phantom field.
func isReservedFieldName(field string) bool {
	return field == "__fields__"
}

// splitRecordKey decomposes a key into (hashID, field) when it is a record
// property key — `file:<rid>/<field>`, split at the FIRST slash so CID-rooted
// and UUID-rooted records parse alike.
func splitRecordKey(key string) (hashID, field string, ok bool) {
	rest, found := strings.CutPrefix(key, recordNS)
	if !found {
		return "", "", false
	}
	hashID, field, found = strings.Cut(rest, "/")
	if !found || hashID == "" || field == "" || isReservedFieldName(field) {
		return "", "", false
	}
	return hashID, field, true
}

// isInternalKey reports keys that are derived bookkeeping, which the raw KV
// editor must never write (it would desynchronise the derived state).
func isInternalKey(key string) bool {
	return strings.HasPrefix(key, "idx:") || strings.HasPrefix(key, dupNS) ||
		strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS)
}

// Derived counters, maintained by onChange in the same transaction as the
// change they count (kv.Tx.AddCounter): the root count behind search's `total`
// and /api/stats, and the per-field schema behind /api/schema.
const (
	rootsCountKey  = "idx:count/roots"
	countersReady  = "idx:count/ready"
	counterRebuild = 5000
)

// onChange keeps the derived counters exact. It sees only real changes (kv
// skips unchanged Puts) with the old and new values, so a modified field moves
// one count from its old label to its new one.
func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) {
	if strings.HasPrefix(k, rootsNS) {
		switch {
		case hasNew && !hadOld:
			tx.AddCounter(rootsCountKey, 1)
		case hadOld && !hasNew:
			tx.AddCounter(rootsCountKey, -1)
		}
		return
	}
	if !strings.HasPrefix(k, recordNS) {
		return
	}
	if hadOld {
		if ck, ok := schema.CounterKey(k, old); ok {
			tx.AddCounter(ck, -1)
		}
	}
	if hasNew {
		if ck, ok := schema.CounterKey(k, v); ok {
			tx.AddCounter(ck, 1)
		}
	}
}

// EnsureCounters builds the derived counters when they have never been built —
// a store created before they existed, or one restored from outside meta-core.
// Blocking and single-transaction (writes wait for it; ~1–2 min at millions of
// keys, once), because a build racing live deltas would double-count.
func (c *Client) EnsureCounters() (bool, error) {
	s, err := c.kv()
	if err != nil {
		return false, err
	}
	ctx, cancel := opCtx()
	_, ready, err := s.Get(ctx, countersReady)
	cancel()
	if err != nil || ready {
		return false, err
	}
	return true, c.RebuildCounters()
}

// RebuildCounters recomputes every derived counter from the records, in one
// transaction (writers wait for it).
func (c *Client) RebuildCounters() error {
	s, err := c.kv()
	if err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()
	return s.Update(ctx, func(tx *kv.Tx) error {
		for _, p := range []string{schema.CounterPrefix, rootsCountKey} {
			if _, err := tx.DeletePrefix(p, 0); err != nil {
				return err
			}
		}
		roots, err := tx.Count(tx.Context(), rootsNS)
		if err != nil {
			return err
		}
		tx.AddCounter(rootsCountKey, roots)
		after := ""
		for {
			rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild)
			if err != nil {
				return err
			}
			for _, row := range rows {
				if ck, ok := schema.CounterKey(row.K, row.V); ok {
					tx.AddCounter(ck, 1)
				}
			}
			if len(rows) < counterRebuild {
				break
			}
			after = rows[len(rows)-1].K
		}
		return tx.Put(countersReady, "1")
	})
}

// readRecord returns a record's flat fields (nil when it has none).
func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) {
	prefix := recordPrefix(hashID)
	rows, err := r.Range(ctx, prefix, "", 0)
	if err != nil {
		return nil, fmt.Errorf("read %s: %w", hashID, err)
	}
	if len(rows) == 0 {
		return nil, nil
	}
	out := make(map[string]string, len(rows))
	for _, row := range rows {
		out[row.K[len(prefix):]] = row.V
	}
	return out, nil
}

// writeFieldsTx is the one write path for record fields: the keys, the root
// index, the CID reverse aliases and the search row, all in the caller's
// transaction.
func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error {
	if hashID == "" {
		return fmt.Errorf("empty hash id")
	}
	prefix := recordPrefix(hashID)
	searchChanged := false
	for field, value := range metadata {
		if isReservedFieldName(field) {
			continue
		}
		changed, err := tx.Set(prefix+field, value)
		if err != nil {
			return err
		}
		searchChanged = searchChanged || (changed && affectsHaystack(field))
	}
	newRoot, err := tx.Set(rootKey(hashID), "")
	if err != nil {
		return err
	}
	for field := range metadata {
		if cidStr := cidFromKeysetField(field); cidStr != "" {
			if err := addAliasTx(tx, hashID, cidStr); err != nil {
				return err
			}
		}
	}
	// The FTS row is rewritten only when the haystack can have changed — a
	// re-persisted record whose values are all in place costs reads only.
	if searchChanged || newRoot {
		return refreshSearchTx(tx, hashID)
	}
	return nil
}

// GetMetadataFlat retrieves all metadata for a file as a flat map.
func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	return readRecord(ctx, s, hashID)
}

// SetMetadataFlat stores metadata for a file using flat keys (merge: fields
// absent from metadata are kept — the Redis implementation never cleared
// either).
func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error {
	if len(metadata) == 0 {
		return nil
	}
	return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) })
}

// MergeMetadataFlat merges new metadata into existing (PATCH semantics): new
// keys are added, existing keys are updated, missing keys are NOT deleted.
func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) {
	if len(metadata) == 0 {
		return 0, nil
	}
	if err := c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }); err != nil {
		return 0, err
	}
	return len(metadata), nil
}

// GetAllHashIDs returns every root id, sorted.
func (c *Client) GetAllHashIDs() ([]string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
	defer cancel()
	var out []string
	err = s.ScanPrefix(ctx, rootsNS, readerPage, func(rows []kv.KV) error {
		for _, row := range rows {
			out = append(out, row.K[len(rootsNS):])
		}
		return nil
	})
	return out, err
}

// GetProperty retrieves a single property value ("" when absent).
func (c *Client) GetProperty(hashID, property string) (string, error) {
	s, err := c.kv()
	if err != nil {
		return "", err
	}
	ctx, cancel := opCtx()
	defer cancel()
	v, _, err := s.Get(ctx, recordPrefix(hashID)+property)
	return v, err
}

// SetProperty sets a single property value.
func (c *Client) SetProperty(hashID, property, value string) error {
	if isReservedFieldName(property) {
		return fmt.Errorf("field name %q is reserved", property)
	}
	return c.update(func(tx *kv.Tx) error {
		return writeFieldsTx(tx, hashID, map[string]string{property: value})
	})
}

// deleteRecordTx removes everything keyed off hashID: its properties, the
// reverse-index entries its cids/ members own, its duplicates, its root-index
// entry and its search row. Returns the number of property keys removed.
func deleteRecordTx(tx *kv.Tx, hashID string) (int64, error) {
	ctx := tx.Context()
	cids, err := cidsForRoot(ctx, tx, hashID)
	if err != nil {
		return 0, err
	}
	for _, cidStr := range cids {
		if _, err := tx.Delete(CIDIndexPrefix + cidStr); err != nil {
			return 0, err
		}
	}
	n, err := tx.DeletePrefix(recordPrefix(hashID), 0)
	if err != nil {
		return 0, err
	}
	if _, err := tx.DeletePrefix(dupPrefix(hashID), 0); err != nil {
		return n, err
	}
	if _, err := tx.Delete(rootKey(hashID)); err != nil {
		return n, err
	}
	return n, deleteSearchRowTx(tx, hashID)
}

// DeleteMetadata deletes all metadata for a file, and unmaps every
// reverse-index entry registered for this root, so a subsequent
// GET /api/meta/<old-cid> correctly returns 404 instead of pointing at a root
// with no keys.
func (c *Client) DeleteMetadata(hashID string) (int64, error) {
	var n int64
	err := c.update(func(tx *kv.Tx) error {
		var err error
		n, err = deleteRecordTx(tx, hashID)
		return err
	})
	return n, err
}

// CountFiles returns the number of indexed roots.
func (c *Client) CountFiles() (int, error) {
	n, err := c.CountRoots()
	return int(n), err
}

// LookupPathByCID resolves a bare CID to its file path on disk via the
// reverse index: one lookup followed by a single read of file:<rid>/filePath.
//
// Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS
// CID, a btih, etc. (they all alias to the same root). The bare CID is what
// the editor passes to /api/file/{cid}/info and what plugins write (e.g.
// `backdrop=<bare-cid>`).
//
// Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is
// resolved through this same path: the plugin output directory is a
// watcher root (see config.DefaultWatcherPaths), so every plugin artefact
// gets a midhash alias and resolves through the reverse index like any
// other file. There is no path-companion sidecar field.
func (c *Client) LookupPathByCID(cidStr string) (string, error) {
	uuid, err := c.GetByCID(cidStr)
	if err != nil {
		return "", err
	}
	if uuid == "" {
		return "", nil
	}
	fp, err := c.GetProperty(uuid, "filePath")
	if err != nil {
		return "", err
	}
	return normalizeFilesRelativePath(fp), nil
}

// normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
// safely filepath.Join(FilesPath, …) without double-prefixing. Different
// plugins store paths inconsistently — poster/backdrop use FilesPath-relative
// ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…").
func normalizeFilesRelativePath(p string) string {
	return strings.TrimPrefix(p, "/files")
}

// DeleteProperty deletes a single property.
//
// For a cids/<cid> member this is the full inverse of the write: the flat key
// AND the cid:<cid> reverse alias both go. Anything less leaves the CID still
// resolving to this record — see removeAliasTx.
func (c *Client) DeleteProperty(hashID, property string) error {
	return c.update(func(tx *kv.Tx) error {
		if _, err := tx.Delete(recordPrefix(hashID) + property); err != nil {
			return err
		}
		if cidStr := cidFromKeysetField(property); cidStr != "" {
			if err := removeAliasTx(tx, hashID, cidStr); err != nil {
				return fmt.Errorf("remove alias for %s: %w", cidStr, err)
			}
		}
		if affectsHaystack(property) {
			return refreshSearchTx(tx, hashID)
		}
		return nil
	})
}

// AddToSet adds a value to a set-type field (stored as a pipe-delimited string
// in one flat key). Returns true if value was added, false if it already
// existed.
func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
	added := false
	err := c.update(func(tx *kv.Tx) error {
		current, _, err := tx.Get(tx.Context(), recordPrefix(hashID)+property)
		if err != nil {
			return err
		}
		var values []string
		if current != "" {
			values = strings.Split(current, "|")
		}
		for _, v := range values {
			if v == value {
				return nil
			}
		}
		added = true
		return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(append(values, value), "|")})
	})
	return added, err
}

// RemoveFromSet removes a value from a set-type field (stored in one flat key).
// Returns true if value was removed, false if it didn't exist.
func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
	removed := false
	err := c.update(func(tx *kv.Tx) error {
		key := recordPrefix(hashID) + property
		current, ok, err := tx.Get(tx.Context(), key)
		if err != nil || !ok {
			return err
		}
		values := strings.Split(current, "|")
		kept := make([]string, 0, len(values))
		for _, v := range values {
			if v == value {
				removed = true
			} else {
				kept = append(kept, v)
			}
		}
		if !removed {
			return nil
		}
		if len(kept) == 0 {
			if _, err := tx.Delete(key); err != nil {
				return err
			}
			if affectsHaystack(property) {
				return refreshSearchTx(tx, hashID)
			}
			return nil
		}
		return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(kept, "|")})
	})
	return removed, err
}

// FileTuple holds the (uuid, path, size, mtime) record for one file. Used by
// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple
// matches what's already stored.
type FileTuple struct {
	HashID    string
	FilePath  string
	Size      int64
	MtimeNano int64
}

// GetTuplesForAllFiles returns one FileTuple per indexed root, in batched
// multi-key reads. Skips files missing any of filePath, sizeByte, mtimeNano.
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
	defer cancel()

	fields := [3]string{"filePath", "sizeByte", "mtimeNano"}
	var out []FileTuple
	err = s.ScanPrefix(ctx, rootsNS, 1000, func(rows []kv.KV) error {
		keys := make([]string, 0, len(rows)*len(fields))
		for _, row := range rows {
			prefix := recordPrefix(row.K[len(rootsNS):])
			for _, f := range fields {
				keys = append(keys, prefix+f)
			}
		}
		vals, err := s.MGet(ctx, keys)
		if err != nil {
			return err
		}
		for _, row := range rows {
			h := row.K[len(rootsNS):]
			prefix := recordPrefix(h)
			filePath, sizeStr, mtimeStr := vals[prefix+fields[0]], vals[prefix+fields[1]], vals[prefix+fields[2]]
			if filePath == "" || sizeStr == "" || mtimeStr == "" {
				continue
			}
			size, err := strconv.ParseInt(sizeStr, 10, 64)
			if err != nil {
				continue
			}
			mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
			if err != nil {
				continue
			}
			out = append(out, FileTuple{HashID: h, FilePath: filePath, Size: size, MtimeNano: mtime})
		}
		return nil
	})
	return out, err
}

// GetMemoryInfo reports the store's footprint for dashboards: the database
// file plus its WAL, human-readable. (Under Redis this was used_memory_human;
// the data no longer lives in RAM, so the disk footprint is the honest figure.)
func (c *Client) GetMemoryInfo() (string, error) {
	s, err := c.kv()
	if err != nil {
		return "", err
	}
	return humanBytes(s.FileSize()), nil
}

// DBSize is the database file plus WAL, in bytes.
func (c *Client) DBSize() int64 {
	s, err := c.kv()
	if err != nil {
		return 0
	}
	return s.FileSize()
}

func humanBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%dB", n)
	}
	div, exp := int64(unit), 0
	for m := n / unit; m >= unit; m /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.2f%c", float64(n)/float64(div), "KMGTPE"[exp])
}

// ClearAllMetadata deletes all file metadata, the root index, the CID reverse
// index, duplicates and search rows. Returns the number of roots deleted.
//
// The CID reverse index MUST be purged too: ResolveRoot consults it, so a
// surviving cid:<cid> entry pointing at a root we just deleted silently
// redirects the next /meta/<cid> write back into that dead root, resurrecting
// a zombie record under an old identity.
//
// Runs as many bounded transactions rather than one: a single transaction over
// millions of rows would hold the writer for minutes and balloon the WAL.
func (c *Client) ClearAllMetadata() (int64, error) {
	roots, err := c.CountRoots()
	if err != nil {
		return 0, err
	}
	for _, prefix := range []string{recordNS, CIDIndexPrefix, dupNS, rootsNS, ftsNS} {
		for {
			var n int64
			if err := c.update(func(tx *kv.Tx) error {
				var err error
				n, err = tx.DeletePrefix(prefix, bulkBatch)
				return err
			}); err != nil {
				return 0, fmt.Errorf("clear %s: %w", prefix, err)
			}
			if n < bulkBatch {
				break
			}
		}
	}
	if err := c.update(func(tx *kv.Tx) error {
		_, err := tx.Exec(`DELETE FROM fts`)
		return err
	}); err != nil {
		return 0, fmt.Errorf("clear search rows: %w", err)
	}
	log.Printf("[Storage] Cleared %d file metadata entries", roots)
	return roots, nil
}

// ScanKeys lists the immediate children at the given prefix, S3-style.
//
// When delimiter is set, keys with a delimiter occurrence after the prefix
// collapse into a branch path (the prefix up to and including that first
// delimiter); the rest are returned as leaves. The walk jumps over each
// branch's whole key range once seen, so listing `file:` costs one read per
// record, not one per key. Stops when leaves+branches reaches maxResults; the
// returned cursor (the next key to examine, "0" when exhausted) continues it.
func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ([]string, []string, string, error) {
	s, err := c.kv()
	if err != nil {
		return nil, nil, "", err
	}
	if maxResults <= 0 {
		maxResults = 500
	}
	from := prefix
	if cursorStr != "" && cursorStr != "0" && cursorStr > prefix {
		from = cursorStr
	}
	end := kv.PrefixEnd(prefix)
	ctx, cancel := opCtx()
	defer cancel()

	var leaves, branches []string
	next := "0"
	op := ">=" // after a leaf the bound becomes exclusive (k > leaf)
	for {
		query := `SELECT k FROM kv WHERE k ` + op + ` ?`
		args := []any{from}
		if end != "" {
			query += ` AND k < ?`
			args = append(args, end)
		}
		var k string
		err := s.DB().QueryRowContext(ctx, query+` ORDER BY k LIMIT 1`, args...).Scan(&k)
		if errors.Is(err, sql.ErrNoRows) {
			break
		}
		if err != nil {
			return nil, nil, "", err
		}
		if len(leaves)+len(branches) >= maxResults {
			next = k
			break
		}
		suffix := k[len(prefix):]
		if delimiter != "" {
			if idx := strings.Index(suffix, delimiter); idx >= 0 {
				branch := prefix + suffix[:idx+len(delimiter)]
				branches = append(branches, branch)
				if from = kv.PrefixEnd(branch); from == "" {
					break
				}
				op = ">="
				continue
			}
		}
		leaves = append(leaves, k)
		from, op = k, ">"
	}
	sort.Strings(leaves)
	sort.Strings(branches)
	return leaves, branches, next, nil
}

// FindMatch is a single value-search hit.
type FindMatch struct {
	Key       string `json:"key"`
	Value     string `json:"value"`
	EntryPath string `json:"entryPath"`
	Field     string `json:"field"`
}

// FindByValue returns keys whose path ends with /<field> (for each field in
// fields) and whose value contains the substring, case-insensitively.
// truncated is true when limit was hit.
//
// EntryPath is the parent prefix up to (and including) the slash before
// field — i.e. the file branch path. Useful for revealing the matched entry
// in the tree.
func (c *Client) FindByValue(contains string, fields []string, limit int) ([]FindMatch, bool, error) {
	s, err := c.kv()
	if err != nil {
		return nil, false, err
	}
	if contains == "" || len(fields) == 0 {
		return nil, false, nil
	}
	if limit <= 0 {
		limit = 200
	}
	ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
	defer cancel()

	needle := strings.ToLower(contains)
	var matches []FindMatch
	truncated := false
	for _, field := range fields {
		field = strings.TrimSpace(field)
		if field == "" || isReservedFieldName(field) {
			continue
		}
		if len(matches) >= limit {
			truncated = true
			break
		}
		suffix := "/" + field
		// Suffix match over every key (records and non-record keys alike, as
		// the SCAN "*/<field>" it replaces did). Value containment is checked in
		// Go: SQLite's lower() folds ASCII only.
		rows, err := s.DB().QueryContext(ctx,
			`SELECT k, v FROM kv WHERE substr(k, -?) = ? AND instr(lower(v), ?) > 0 ORDER BY k`,
			len(suffix), suffix, asciiLower(needle))
		if err != nil {
			return nil, false, err
		}
		for rows.Next() {
			var k, v string
			if err := rows.Scan(&k, &v); err != nil {
				rows.Close()
				return nil, false, err
			}
			if isInternalKey(k) || !strings.Contains(strings.ToLower(v), needle) {
				continue
			}
			matches = append(matches, FindMatch{Key: k, Value: v, EntryPath: strings.TrimSuffix(k, field), Field: field})
			if len(matches) >= limit {
				truncated = true
				break
			}
		}
		rows.Close()
	}
	sort.Slice(matches, func(i, j int) bool { return matches[i].Key < matches[j].Key })
	return matches, truncated, nil
}

// SearchKeys returns up to limit keys whose full path contains the substring,
// in key order. truncated is true when the search stopped at limit.
func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) {
	s, err := c.kv()
	if err != nil {
		return nil, false, err
	}
	if contains == "" {
		return nil, false, nil
	}
	if limit <= 0 {
		limit = 500
	}
	ctx, cancel := opCtx()
	defer cancel()

	rows, err := s.DB().QueryContext(ctx, `SELECT k FROM kv WHERE instr(k, ?) > 0 ORDER BY k LIMIT ?`, contains, limit+1)
	if err != nil {
		return nil, false, err
	}
	defer rows.Close()
	results := make([]string, 0, limit)
	for rows.Next() {
		var k string
		if err := rows.Scan(&k); err != nil {
			return nil, false, err
		}
		results = append(results, k)
	}
	if err := rows.Err(); err != nil {
		return nil, false, err
	}
	if len(results) > limit {
		return results[:limit], true, nil
	}
	return results, false, nil
}

// GetKeyInfo returns the key's type — "string" when it exists (every row is a
// string now), with its value. exists is false when the key is absent.
func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, err error) {
	s, err := c.kv()
	if err != nil {
		return "", "", false, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	v, ok, err := s.Get(ctx, key)
	if err != nil || !ok {
		return "", "", false, err
	}
	return "string", v, true, nil
}

// SetRawValue writes a string value at the exact key (the raw KV editor).
// Refuses derived bookkeeping keys. A record key keeps the search row honest
// like every other write path.
func (c *Client) SetRawValue(key, value string) error {
	if isInternalKey(key) {
		return fmt.Errorf("key %q is internal bookkeeping and not editable", key)
	}
	return c.update(func(tx *kv.Tx) error {
		if err := tx.Put(key, value); err != nil {
			return err
		}
		if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
			return refreshSearchTx(tx, hashID)
		}
		return nil
	})
}

// DeleteRawKey deletes the exact key. Returns true if the key existed.
func (c *Client) DeleteRawKey(key string) (bool, error) {
	if isInternalKey(key) {
		return false, fmt.Errorf("key %q is internal bookkeeping and not editable", key)
	}
	existed := false
	err := c.update(func(tx *kv.Tx) error {
		var err error
		if existed, err = tx.Delete(key); err != nil || !existed {
			return err
		}
		if hashID, field, ok := splitRecordKey(key); ok && affectsHaystack(field) {
			return refreshSearchTx(tx, hashID)
		}
		return nil
	})
	return existed, err
}

// IndexAdd adds a hash ID to the root index. Idempotent.
func (c *Client) IndexAdd(hashID string) error {
	return c.update(func(tx *kv.Tx) error {
		if err := tx.Put(rootKey(hashID), ""); err != nil {
			return err
		}
		return ensureSearchRowTx(tx, hashID)
	})
}

// IndexRemoveIfEmpty removes hashID from the root index (and drops its search
// row) when no file:<hashID>/* keys remain. Returns true if it was removed.
func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
	removed := false
	err := c.update(func(tx *kv.Tx) error {
		n, err := tx.Count(tx.Context(), recordPrefix(hashID))
		if err != nil || n > 0 {
			return err
		}
		if removed, err = tx.Delete(rootKey(hashID)); err != nil {
			return err
		}
		return deleteSearchRowTx(tx, hashID)
	})
	return removed, err
}

// GetPrefix returns the key namespace prefix. Always empty: meta-core owns its
// database file outright, so there is nothing to share a keyspace with.
func (c *Client) GetPrefix() string { return "" }

// XAdd appends an event to a stream. maxLen > 0 trims to about that many
// entries; maxLen <= 0 keeps everything.
func (c *Client) XAdd(stream string, maxLen int64, fields map[string]interface{}) (string, error) {
	values := make(map[string]string, len(fields))
	for k, v := range fields {
		values[k] = fmt.Sprint(v)
	}
	var id string
	err := c.update(func(tx *kv.Tx) error {
		var err error
		id, err = tx.Append(stream, maxLen, values)
		return err
	})
	return id, err
}

// AppendEvents appends a batch of entries to a stream in one transaction.
func (c *Client) AppendEvents(stream string, maxLen int64, entries []map[string]string) error {
	if len(entries) == 0 {
		return nil
	}
	return c.update(func(tx *kv.Tx) error {
		for _, e := range entries {
			if _, err := tx.Append(stream, 0, e); err != nil {
				return err
			}
		}
		if maxLen > 0 {
			return tx.TrimAll(stream, maxLen)
		}
		return nil
	})
}

// ClearStream deletes every entry of a stream (its ID high-water mark
// survives, so IDs keep increasing across the clear).
func (c *Client) ClearStream(stream string) error {
	return c.update(func(tx *kv.Tx) error { return tx.ClearStream(stream) })
}

// StreamEntry is one stream entry.
type StreamEntry struct {
	ID     string
	Values map[string]interface{}
}

// XRevRange reads stream entries from newest to oldest. start must be "+" or
// empty (newest); stop is "-"/empty (oldest) or "(<id>" for an exclusive lower
// bound — the only shapes callers use. count > 0 caps the result.
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	if start != "" && start != "+" {
		return nil, fmt.Errorf("XRevRange: unsupported start %q", start)
	}
	after := ""
	switch {
	case stop == "" || stop == "-":
	case strings.HasPrefix(stop, "("):
		after = stop[1:]
	default:
		return nil, fmt.Errorf("XRevRange: unsupported stop %q", stop)
	}
	ctx, cancel := opCtx()
	defer cancel()
	entries, err := s.RevRange(ctx, stream, after, int(count))
	if err != nil {
		return nil, err
	}
	out := make([]StreamEntry, 0, len(entries))
	for _, e := range entries {
		values := make(map[string]interface{}, len(e.Values))
		for k, v := range e.Values {
			values[k] = v
		}
		out = append(out, StreamEntry{ID: e.ID, Values: values})
	}
	return out, nil
}

// GetStreamLength returns the number of surviving entries of a stream.
func (c *Client) GetStreamLength(stream string) (int64, error) {
	s, err := c.kv()
	if err != nil {
		return 0, err
	}
	ctx, cancel := opCtx()
	defer cancel()
	return s.Len(ctx, stream)
}