881 lines
25 KiB
Go
881 lines
25 KiB
Go
package storage
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/metazla/meta-core/internal/kv"
|
||
"github.com/metazla/meta-core/internal/schema"
|
||
)
|
||
|
||
// meta-core keeps two SQLite files (docs/implementation/metacore-claims-model.md):
|
||
//
|
||
// - the TRUTH database (truth.go): principal, claim, cid_group, meta — the
|
||
// signed votes and the locally minted rid names, primary keys only, backed
|
||
// up;
|
||
//
|
||
// - the CACHE database: everything derived from truth, never backed up,
|
||
// rebuilt when missing (apply.go). Its kv table keeps the v1 layout:
|
||
//
|
||
// file:<rid>/<field> the MERGED record view — the ONLY thing under `file:`
|
||
// cid:<cid> cid → rid lookup (copy of truth's cid_group)
|
||
// idx:roots/<rid> the set of every record
|
||
// idx:fts/<rid> rowid of the record's row in the FTS table
|
||
// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange
|
||
// ev:<stream>/…, evhwm:… event streams, see internal/kv/stream.go
|
||
// meta:applied_seq, meta:rebuilt_at apply-loop bookkeeping
|
||
// anything else service scratch KV (btmagnet:*, searchcov:*), /api/kv/value
|
||
//
|
||
// plus the tables claim_by_pid and local_file (localfile.go), and the
|
||
// field index + FTS (search.go).
|
||
//
|
||
// Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is
|
||
// exactly the merged records, so a record is the key range [file:<rid>/,
|
||
// file:<rid>0), the field expression index covers only real fields, and every
|
||
// change under `file:` is a real field change worth publishing on meta:events.
|
||
const (
|
||
recordNS = "file:"
|
||
rootsNS = "idx:roots/"
|
||
ftsNS = "idx:fts/"
|
||
dupNS = "dup:"
|
||
streamNS = "ev:"
|
||
hwmNS = "evhwm:"
|
||
opTimeout = 30 * time.Second
|
||
bulkBatch = 10_000
|
||
readerPage = 5_000
|
||
)
|
||
|
||
// MetaEventsStream receives a set/del entry for every merged-record change. It
|
||
// is written by the cache engine in the same transaction as the change, and
|
||
// served as SSE by /api/events/meta.
|
||
const MetaEventsStream = "meta:events"
|
||
|
||
// MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is
|
||
// a contract requirement for the SSE wire — see docs/api-mediated-access.md
|
||
// "Backing-store retention and lifecycle".
|
||
const MetaEventsMaxLen = 100_000
|
||
|
||
// Client is meta-core's metadata store: the truth database, the cache, and
|
||
// the apply loop between them.
|
||
type Client struct {
|
||
store *kv.Store // cache
|
||
truth *kv.Store
|
||
applier *applier
|
||
mu sync.RWMutex
|
||
|
||
keysMu sync.Mutex
|
||
keysCount int64
|
||
keysAt time.Time
|
||
}
|
||
|
||
// NewClient creates an unconnected client.
|
||
func NewClient() *Client {
|
||
return &Client{}
|
||
}
|
||
|
||
// Connect opens (creating if needed) the truth database at truthPath and the
|
||
// cache at cachePath. Call EnsureSchemaVersion, then Start.
|
||
func (c *Client) Connect(truthPath, cachePath string) error {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
|
||
truth, err := kv.Open(truthPath, kv.Options{
|
||
NoKVTable: true,
|
||
Schema: truthSchema,
|
||
WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
|
||
ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
|
||
Readers: envInt("META_CORE_DB_READERS", 8),
|
||
})
|
||
if err != nil {
|
||
return fmt.Errorf("open truth %s: %w", truthPath, err)
|
||
}
|
||
store, err := kv.Open(cachePath, kv.Options{
|
||
EventStream: MetaEventsStream,
|
||
EventMaxLen: MetaEventsMaxLen,
|
||
EventKey: func(k string) bool { return strings.HasPrefix(k, recordNS) },
|
||
WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64),
|
||
ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16),
|
||
Readers: envInt("META_CORE_DB_READERS", 8),
|
||
Schema: searchSchema + cacheTablesSchema,
|
||
OnChange: onChange,
|
||
Synchronous: "OFF",
|
||
})
|
||
if err != nil {
|
||
truth.Close()
|
||
return fmt.Errorf("open cache %s: %w", cachePath, err)
|
||
}
|
||
c.truth, c.store = truth, store
|
||
log.Printf("[Storage] Opened truth %s, cache %s", truthPath, cachePath)
|
||
return nil
|
||
}
|
||
|
||
// Start brings the cache level with truth (replay or rebuild) and starts the
|
||
// apply loop. Writes before Start fail.
|
||
func (c *Client) Start() error {
|
||
applied, err := c.catchUp()
|
||
if err != nil {
|
||
return fmt.Errorf("cache catch-up: %w", err)
|
||
}
|
||
c.mu.Lock()
|
||
c.applier = newApplier(c, applied)
|
||
c.mu.Unlock()
|
||
return nil
|
||
}
|
||
|
||
// Close stops the apply loop and closes both databases.
|
||
func (c *Client) Close() error {
|
||
c.mu.Lock()
|
||
defer c.mu.Unlock()
|
||
if c.applier != nil {
|
||
c.applier.close()
|
||
c.applier = nil
|
||
}
|
||
var errs []error
|
||
if c.store != nil {
|
||
errs = append(errs, c.store.Close())
|
||
c.store = nil
|
||
}
|
||
if c.truth != nil {
|
||
errs = append(errs, c.truth.Close())
|
||
c.truth = nil
|
||
}
|
||
return errors.Join(errs...)
|
||
}
|
||
|
||
// IsConnected reports whether the databases are open.
|
||
func (c *Client) IsConnected() bool {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
return c.store != nil && c.truth != nil
|
||
}
|
||
|
||
// Health checks that both databases answer.
|
||
func (c *Client) Health() bool {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
t, err := c.truthStore()
|
||
if err != nil {
|
||
return false
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||
defer cancel()
|
||
return s.Ping(ctx) == nil && t.Ping(ctx) == nil
|
||
}
|
||
|
||
// Store exposes the cache engine (event streams for SSE and the schema
|
||
// indexer).
|
||
func (c *Client) Store() *kv.Store {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
return c.store
|
||
}
|
||
|
||
func (c *Client) kv() (*kv.Store, error) {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
if c.store == nil {
|
||
return nil, fmt.Errorf("not connected")
|
||
}
|
||
return c.store, nil
|
||
}
|
||
|
||
func (c *Client) truthStore() (*kv.Store, error) {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
if c.truth == nil || c.applier == nil {
|
||
return nil, fmt.Errorf("not connected")
|
||
}
|
||
return c.truth, nil
|
||
}
|
||
|
||
// update runs fn in one cache write transaction with the standard timeout.
|
||
func (c *Client) update(fn func(tx *kv.Tx) error) error {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), opTimeout)
|
||
defer cancel()
|
||
return s.Update(ctx, fn)
|
||
}
|
||
|
||
func opCtx() (context.Context, context.CancelFunc) {
|
||
return context.WithTimeout(context.Background(), opTimeout)
|
||
}
|
||
|
||
func recordPrefix(hashID string) string { return recordNS + hashID + "/" }
|
||
func rootKey(hashID string) string { return rootsNS + hashID }
|
||
func dupPrefix(hashID string) string { return dupNS + hashID + "/" }
|
||
|
||
// isReservedFieldName reports whether field names bookkeeping rather than
|
||
// metadata. `__fields__` was the Redis-era per-record field index; it no
|
||
// longer exists, but a payload carrying it (an old snapshot) is still refused
|
||
// so it can never become a phantom field.
|
||
func isReservedFieldName(field string) bool {
|
||
return field == "__fields__"
|
||
}
|
||
|
||
// splitRecordKey decomposes a key into (hashID, field) when it is a record
|
||
// property key — `file:<rid>/<field>`, split at the FIRST slash so CID-rooted
|
||
// and UUID-rooted records parse alike.
|
||
func splitRecordKey(key string) (hashID, field string, ok bool) {
|
||
rest, found := strings.CutPrefix(key, recordNS)
|
||
if !found {
|
||
return "", "", false
|
||
}
|
||
hashID, field, found = strings.Cut(rest, "/")
|
||
if !found || hashID == "" || field == "" || isReservedFieldName(field) {
|
||
return "", "", false
|
||
}
|
||
return hashID, field, true
|
||
}
|
||
|
||
// isInternalKey reports keys that are derived bookkeeping, which the raw KV
|
||
// editor must never write (it would desynchronise the derived state).
|
||
func isInternalKey(key string) bool {
|
||
for _, p := range []string{recordNS, CIDIndexPrefix, "idx:", dupNS, streamNS, hwmNS, "meta:", "meta-core:"} {
|
||
if strings.HasPrefix(key, p) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// SetServiceValue writes service scratch KV (btmagnet:*, searchcov:*) — the
|
||
// only raw key writes left. Merged records and bookkeeping are refused.
|
||
func (c *Client) SetServiceValue(key, value string) error {
|
||
if key == "" || isInternalKey(key) {
|
||
return fmt.Errorf("key %q is not a service key", key)
|
||
}
|
||
return c.update(func(tx *kv.Tx) error { return tx.Put(key, value) })
|
||
}
|
||
|
||
// DeleteServiceValue deletes a service scratch key. Returns true if it existed.
|
||
func (c *Client) DeleteServiceValue(key string) (bool, error) {
|
||
if key == "" || isInternalKey(key) {
|
||
return false, fmt.Errorf("key %q is not a service key", key)
|
||
}
|
||
existed := false
|
||
err := c.update(func(tx *kv.Tx) error {
|
||
var err error
|
||
existed, err = tx.Delete(key)
|
||
return err
|
||
})
|
||
return existed, err
|
||
}
|
||
|
||
// ClearAllMetadata wipes BOTH databases: every claim, signer and group in
|
||
// truth, and every derived row of the cache. Irreversible — truth holds votes
|
||
// nothing else can recover. Returns the number of records that existed.
|
||
func (c *Client) ClearAllMetadata() (int64, error) {
|
||
truth, err := c.truthStore()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
roots, err := c.CountRoots()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||
defer cancel()
|
||
if err := truth.Update(ctx, func(tx *kv.Tx) error {
|
||
for _, t := range []string{"claim", "principal", "cid_group"} {
|
||
if _, err := tx.Exec(`DELETE FROM ` + t); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
_, err := tx.Exec(`DELETE FROM meta WHERE name = ?`, lastSeqName)
|
||
return err
|
||
}); err != nil {
|
||
return 0, fmt.Errorf("clear truth: %w", err)
|
||
}
|
||
for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} {
|
||
for {
|
||
var n int64
|
||
if err := c.update(func(tx *kv.Tx) error {
|
||
var err error
|
||
n, err = tx.DeletePrefix(prefix, bulkBatch)
|
||
return err
|
||
}); err != nil {
|
||
return 0, fmt.Errorf("clear %s: %w", prefix, err)
|
||
}
|
||
if n < bulkBatch {
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if err := c.update(func(tx *kv.Tx) error {
|
||
for _, q := range []string{`DELETE FROM fts`, `DELETE FROM claim_by_pid`, `DELETE FROM local_file`} {
|
||
if _, err := tx.Exec(q); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return tx.Put(appliedSeqKey, "0")
|
||
}); err != nil {
|
||
return 0, fmt.Errorf("clear cache: %w", err)
|
||
}
|
||
c.applier.mu.Lock()
|
||
c.applier.applied = 0
|
||
c.applier.mu.Unlock()
|
||
log.Printf("[Storage] Cleared both databases (%d records)", roots)
|
||
return roots, nil
|
||
}
|
||
|
||
// Derived counters, maintained by onChange in the same transaction as the
|
||
// change they count (kv.Tx.AddCounter): the root count behind search's `total`
|
||
// and /api/stats, and the per-field schema behind /api/schema.
|
||
const (
|
||
rootsCountKey = "idx:count/roots"
|
||
countersReady = "idx:count/ready"
|
||
counterRebuild = 5000
|
||
)
|
||
|
||
// onChange keeps the derived counters exact. It sees only real changes (kv
|
||
// skips unchanged Puts) with the old and new values, so a modified field moves
|
||
// one count from its old label to its new one.
|
||
func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) {
|
||
if strings.HasPrefix(k, rootsNS) {
|
||
switch {
|
||
case hasNew && !hadOld:
|
||
tx.AddCounter(rootsCountKey, 1)
|
||
case hadOld && !hasNew:
|
||
tx.AddCounter(rootsCountKey, -1)
|
||
}
|
||
return
|
||
}
|
||
if !strings.HasPrefix(k, recordNS) {
|
||
return
|
||
}
|
||
if hadOld {
|
||
if ck, ok := schema.CounterKey(k, old); ok {
|
||
tx.AddCounter(ck, -1)
|
||
}
|
||
}
|
||
if hasNew {
|
||
if ck, ok := schema.CounterKey(k, v); ok {
|
||
tx.AddCounter(ck, 1)
|
||
}
|
||
}
|
||
}
|
||
|
||
// EnsureCounters builds the derived counters when they have never been built —
|
||
// a store created before they existed, or one restored from outside meta-core.
|
||
// Blocking and single-transaction (writes wait for it; ~1–2 min at millions of
|
||
// keys, once), because a build racing live deltas would double-count.
|
||
func (c *Client) EnsureCounters() (bool, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
ctx, cancel := opCtx()
|
||
_, ready, err := s.Get(ctx, countersReady)
|
||
cancel()
|
||
if err != nil || ready {
|
||
return false, err
|
||
}
|
||
return true, c.RebuildCounters()
|
||
}
|
||
|
||
// RebuildCounters recomputes every derived counter from the records, in one
|
||
// transaction (writers wait for it).
|
||
func (c *Client) RebuildCounters() error {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
||
defer cancel()
|
||
return s.Update(ctx, func(tx *kv.Tx) error {
|
||
for _, p := range []string{schema.CounterPrefix, rootsCountKey} {
|
||
if _, err := tx.DeletePrefix(p, 0); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
roots, err := tx.Count(tx.Context(), rootsNS)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
tx.AddCounter(rootsCountKey, roots)
|
||
after := ""
|
||
for {
|
||
rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, row := range rows {
|
||
if ck, ok := schema.CounterKey(row.K, row.V); ok {
|
||
tx.AddCounter(ck, 1)
|
||
}
|
||
}
|
||
if len(rows) < counterRebuild {
|
||
break
|
||
}
|
||
after = rows[len(rows)-1].K
|
||
}
|
||
return tx.Put(countersReady, "1")
|
||
})
|
||
}
|
||
|
||
// readRecord returns a record's flat fields (nil when it has none).
|
||
func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) {
|
||
prefix := recordPrefix(hashID)
|
||
rows, err := r.Range(ctx, prefix, "", 0)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("read %s: %w", hashID, err)
|
||
}
|
||
if len(rows) == 0 {
|
||
return nil, nil
|
||
}
|
||
out := make(map[string]string, len(rows))
|
||
for _, row := range rows {
|
||
out[row.K[len(prefix):]] = row.V
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// GetMetadataFlat retrieves all metadata for a file as a flat map.
|
||
func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
return readRecord(ctx, s, hashID)
|
||
}
|
||
|
||
// GetAllHashIDs returns every root id, sorted.
|
||
func (c *Client) GetAllHashIDs() ([]string, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
|
||
defer cancel()
|
||
var out []string
|
||
err = s.ScanPrefix(ctx, rootsNS, readerPage, func(rows []kv.KV) error {
|
||
for _, row := range rows {
|
||
out = append(out, row.K[len(rootsNS):])
|
||
}
|
||
return nil
|
||
})
|
||
return out, err
|
||
}
|
||
|
||
// GetProperty retrieves a single property value ("" when absent).
|
||
func (c *Client) GetProperty(hashID, property string) (string, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
v, _, err := s.Get(ctx, recordPrefix(hashID)+property)
|
||
return v, err
|
||
}
|
||
|
||
// CountFiles returns the number of indexed roots.
|
||
func (c *Client) CountFiles() (int, error) {
|
||
n, err := c.CountRoots()
|
||
return int(n), err
|
||
}
|
||
|
||
// LookupPathByCID resolves a bare CID to its file path on disk via the
|
||
// reverse index: one lookup followed by a single read of file:<rid>/filePath.
|
||
//
|
||
// Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS
|
||
// CID, a btih, etc. (they all alias to the same root). The bare CID is what
|
||
// the editor passes to /api/file/{cid}/info and what plugins write (e.g.
|
||
// `backdrop=<bare-cid>`).
|
||
//
|
||
// Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is
|
||
// resolved through this same path: the plugin output directory is a
|
||
// watcher root (see config.DefaultWatcherPaths), so every plugin artefact
|
||
// gets a midhash alias and resolves through the reverse index like any
|
||
// other file. There is no path-companion sidecar field.
|
||
func (c *Client) LookupPathByCID(cidStr string) (string, error) {
|
||
uuid, err := c.GetByCID(cidStr)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if uuid == "" {
|
||
return "", nil
|
||
}
|
||
fp, err := c.GetProperty(uuid, "filePath")
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return normalizeFilesRelativePath(fp), nil
|
||
}
|
||
|
||
// normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
|
||
// safely filepath.Join(FilesPath, …) without double-prefixing. Different
|
||
// plugins store paths inconsistently — poster/backdrop use FilesPath-relative
|
||
// ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…").
|
||
func normalizeFilesRelativePath(p string) string {
|
||
return strings.TrimPrefix(p, "/files")
|
||
}
|
||
|
||
// DeleteProperty deletes a single property.
|
||
//
|
||
|
||
// GetMemoryInfo reports the store's footprint for dashboards: the database
|
||
// file plus its WAL, human-readable. (Under Redis this was used_memory_human;
|
||
// the data no longer lives in RAM, so the disk footprint is the honest figure.)
|
||
func (c *Client) GetMemoryInfo() (string, error) {
|
||
if _, err := c.kv(); err != nil {
|
||
return "", err
|
||
}
|
||
return humanBytes(c.DBSize()), nil
|
||
}
|
||
|
||
// DBSize is both database files plus their WALs, in bytes.
|
||
func (c *Client) DBSize() int64 {
|
||
c.mu.RLock()
|
||
defer c.mu.RUnlock()
|
||
var n int64
|
||
for _, st := range []*kv.Store{c.store, c.truth} {
|
||
if st != nil {
|
||
n += st.FileSize()
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
func humanBytes(n int64) string {
|
||
const unit = 1024
|
||
if n < unit {
|
||
return fmt.Sprintf("%dB", n)
|
||
}
|
||
div, exp := int64(unit), 0
|
||
for m := n / unit; m >= unit; m /= unit {
|
||
div *= unit
|
||
exp++
|
||
}
|
||
return fmt.Sprintf("%.2f%c", float64(n)/float64(div), "KMGTPE"[exp])
|
||
}
|
||
|
||
// ClearAllMetadata deletes all file metadata, the root index, the CID reverse
|
||
// index, duplicates and search rows. Returns the number of roots deleted.
|
||
//
|
||
// The CID reverse index MUST be purged too: ResolveRoot consults it, so a
|
||
// surviving cid:<cid> entry pointing at a root we just deleted silently
|
||
// redirects the next /meta/<cid> write back into that dead root, resurrecting
|
||
// a zombie record under an old identity.
|
||
//
|
||
|
||
// ScanKeys lists the immediate children at the given prefix, S3-style.
|
||
//
|
||
// When delimiter is set, keys with a delimiter occurrence after the prefix
|
||
// collapse into a branch path (the prefix up to and including that first
|
||
// delimiter); the rest are returned as leaves. The walk jumps over each
|
||
// branch's whole key range once seen, so listing `file:` costs one read per
|
||
// record, not one per key. Stops when leaves+branches reaches maxResults; the
|
||
// returned cursor (the next key to examine, "0" when exhausted) continues it.
|
||
func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ([]string, []string, string, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, nil, "", err
|
||
}
|
||
if maxResults <= 0 {
|
||
maxResults = 500
|
||
}
|
||
from := prefix
|
||
if cursorStr != "" && cursorStr != "0" && cursorStr > prefix {
|
||
from = cursorStr
|
||
}
|
||
end := kv.PrefixEnd(prefix)
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
|
||
var leaves, branches []string
|
||
next := "0"
|
||
op := ">=" // after a leaf the bound becomes exclusive (k > leaf)
|
||
for {
|
||
query := `SELECT k FROM kv WHERE k ` + op + ` ?`
|
||
args := []any{from}
|
||
if end != "" {
|
||
query += ` AND k < ?`
|
||
args = append(args, end)
|
||
}
|
||
var k string
|
||
err := s.DB().QueryRowContext(ctx, query+` ORDER BY k LIMIT 1`, args...).Scan(&k)
|
||
if errors.Is(err, sql.ErrNoRows) {
|
||
break
|
||
}
|
||
if err != nil {
|
||
return nil, nil, "", err
|
||
}
|
||
if len(leaves)+len(branches) >= maxResults {
|
||
next = k
|
||
break
|
||
}
|
||
suffix := k[len(prefix):]
|
||
if delimiter != "" {
|
||
if idx := strings.Index(suffix, delimiter); idx >= 0 {
|
||
branch := prefix + suffix[:idx+len(delimiter)]
|
||
branches = append(branches, branch)
|
||
if from = kv.PrefixEnd(branch); from == "" {
|
||
break
|
||
}
|
||
op = ">="
|
||
continue
|
||
}
|
||
}
|
||
leaves = append(leaves, k)
|
||
from, op = k, ">"
|
||
}
|
||
leaves, branches = serviceOnly(leaves), serviceOnly(branches)
|
||
sort.Strings(leaves)
|
||
sort.Strings(branches)
|
||
return leaves, branches, next, nil
|
||
}
|
||
|
||
// serviceOnly drops merged-record and bookkeeping keys: the /api/kv browser
|
||
// shows service scratch KV only.
|
||
func serviceOnly(keys []string) []string {
|
||
out := keys[:0]
|
||
for _, k := range keys {
|
||
if !isInternalKey(k) {
|
||
out = append(out, k)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// FindMatch is a single value-search hit.
|
||
type FindMatch struct {
|
||
Key string `json:"key"`
|
||
Value string `json:"value"`
|
||
EntryPath string `json:"entryPath"`
|
||
Field string `json:"field"`
|
||
}
|
||
|
||
// FindByValue returns keys whose path ends with /<field> (for each field in
|
||
// fields) and whose value contains the substring, case-insensitively.
|
||
// truncated is true when limit was hit.
|
||
//
|
||
// EntryPath is the parent prefix up to (and including) the slash before
|
||
// field — i.e. the file branch path. Useful for revealing the matched entry
|
||
// in the tree.
|
||
func (c *Client) FindByValue(contains string, fields []string, limit int) ([]FindMatch, bool, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
if contains == "" || len(fields) == 0 {
|
||
return nil, false, nil
|
||
}
|
||
if limit <= 0 {
|
||
limit = 200
|
||
}
|
||
ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
|
||
defer cancel()
|
||
|
||
needle := strings.ToLower(contains)
|
||
var matches []FindMatch
|
||
truncated := false
|
||
for _, field := range fields {
|
||
field = strings.TrimSpace(field)
|
||
if field == "" || isReservedFieldName(field) {
|
||
continue
|
||
}
|
||
if len(matches) >= limit {
|
||
truncated = true
|
||
break
|
||
}
|
||
suffix := "/" + field
|
||
// Suffix match over every key (records and non-record keys alike, as
|
||
// the SCAN "*/<field>" it replaces did). Value containment is checked in
|
||
// Go: SQLite's lower() folds ASCII only.
|
||
rows, err := s.DB().QueryContext(ctx,
|
||
`SELECT k, v FROM kv WHERE substr(k, -?) = ? AND instr(lower(v), ?) > 0 ORDER BY k`,
|
||
len(suffix), suffix, asciiLower(needle))
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
for rows.Next() {
|
||
var k, v string
|
||
if err := rows.Scan(&k, &v); err != nil {
|
||
rows.Close()
|
||
return nil, false, err
|
||
}
|
||
if isInternalKey(k) || !strings.Contains(strings.ToLower(v), needle) {
|
||
continue
|
||
}
|
||
matches = append(matches, FindMatch{Key: k, Value: v, EntryPath: strings.TrimSuffix(k, field), Field: field})
|
||
if len(matches) >= limit {
|
||
truncated = true
|
||
break
|
||
}
|
||
}
|
||
rows.Close()
|
||
}
|
||
sort.Slice(matches, func(i, j int) bool { return matches[i].Key < matches[j].Key })
|
||
return matches, truncated, nil
|
||
}
|
||
|
||
// SearchKeys returns up to limit keys whose full path contains the substring,
|
||
// in key order. truncated is true when the search stopped at limit.
|
||
func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
if contains == "" {
|
||
return nil, false, nil
|
||
}
|
||
if limit <= 0 {
|
||
limit = 500
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
|
||
rows, err := s.DB().QueryContext(ctx, `SELECT k FROM kv WHERE instr(k, ?) > 0 ORDER BY k LIMIT ?`, contains, limit+1)
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
defer rows.Close()
|
||
results := make([]string, 0, limit)
|
||
for rows.Next() {
|
||
var k string
|
||
if err := rows.Scan(&k); err != nil {
|
||
return nil, false, err
|
||
}
|
||
results = append(results, k)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, false, err
|
||
}
|
||
truncated := len(results) > limit
|
||
if truncated {
|
||
results = results[:limit]
|
||
}
|
||
return serviceOnly(results), truncated, nil
|
||
}
|
||
|
||
// GetKeyInfo returns the key's type — "string" when it exists (every row is a
|
||
// string now), with its value. exists is false when the key is absent.
|
||
func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, err error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return "", "", false, err
|
||
}
|
||
if isInternalKey(key) {
|
||
return "", "", false, nil
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
v, ok, err := s.Get(ctx, key)
|
||
if err != nil || !ok {
|
||
return "", "", false, err
|
||
}
|
||
return "string", v, true, nil
|
||
}
|
||
|
||
// GetPrefix returns the key namespace prefix. Always empty: meta-core owns its
|
||
// database file outright, so there is nothing to share a keyspace with.
|
||
func (c *Client) GetPrefix() string { return "" }
|
||
|
||
// XAdd appends an event to a stream. maxLen > 0 trims to about that many
|
||
// entries; maxLen <= 0 keeps everything.
|
||
func (c *Client) XAdd(stream string, maxLen int64, fields map[string]interface{}) (string, error) {
|
||
values := make(map[string]string, len(fields))
|
||
for k, v := range fields {
|
||
values[k] = fmt.Sprint(v)
|
||
}
|
||
var id string
|
||
err := c.update(func(tx *kv.Tx) error {
|
||
var err error
|
||
id, err = tx.Append(stream, maxLen, values)
|
||
return err
|
||
})
|
||
return id, err
|
||
}
|
||
|
||
// AppendEvents appends a batch of entries to a stream in one transaction.
|
||
func (c *Client) AppendEvents(stream string, maxLen int64, entries []map[string]string) error {
|
||
if len(entries) == 0 {
|
||
return nil
|
||
}
|
||
return c.update(func(tx *kv.Tx) error {
|
||
for _, e := range entries {
|
||
if _, err := tx.Append(stream, 0, e); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if maxLen > 0 {
|
||
return tx.TrimAll(stream, maxLen)
|
||
}
|
||
return nil
|
||
})
|
||
}
|
||
|
||
// ClearStream deletes every entry of a stream (its ID high-water mark
|
||
// survives, so IDs keep increasing across the clear).
|
||
func (c *Client) ClearStream(stream string) error {
|
||
return c.update(func(tx *kv.Tx) error { return tx.ClearStream(stream) })
|
||
}
|
||
|
||
// StreamEntry is one stream entry.
|
||
type StreamEntry struct {
|
||
ID string
|
||
Values map[string]interface{}
|
||
}
|
||
|
||
// XRevRange reads stream entries from newest to oldest. start must be "+" or
|
||
// empty (newest); stop is "-"/empty (oldest) or "(<id>" for an exclusive lower
|
||
// bound — the only shapes callers use. count > 0 caps the result.
|
||
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if start != "" && start != "+" {
|
||
return nil, fmt.Errorf("XRevRange: unsupported start %q", start)
|
||
}
|
||
after := ""
|
||
switch {
|
||
case stop == "" || stop == "-":
|
||
case strings.HasPrefix(stop, "("):
|
||
after = stop[1:]
|
||
default:
|
||
return nil, fmt.Errorf("XRevRange: unsupported stop %q", stop)
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
entries, err := s.RevRange(ctx, stream, after, int(count))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
out := make([]StreamEntry, 0, len(entries))
|
||
for _, e := range entries {
|
||
values := make(map[string]interface{}, len(e.Values))
|
||
for k, v := range e.Values {
|
||
values[k] = v
|
||
}
|
||
out = append(out, StreamEntry{ID: e.ID, Values: values})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// GetStreamLength returns the number of surviving entries of a stream.
|
||
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
return s.Len(ctx, stream)
|
||
}
|