package storage import ( "context" "database/sql" "errors" "fmt" "log" "sort" "strings" "sync" "time" "github.com/metazla/meta-core/internal/kv" "github.com/metazla/meta-core/internal/schema" ) // meta-core keeps two SQLite files (docs/implementation/metacore-claims-model.md): // // - the TRUTH database (truth.go): principal, claim, cid_group, meta — the // signed votes and the locally minted rid names, primary keys only, backed // up; // // - the CACHE database: everything derived from truth, never backed up, // rebuilt when missing (apply.go). Its kv table keeps the v1 layout: // // file:/ the MERGED record view — the ONLY thing under `file:` // cid: cid → rid lookup (copy of truth's cid_group) // idx:roots/ the set of every record // idx:fts/ rowid of the record's row in the FTS table // idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange // ev:/…, evhwm:… event streams, see internal/kv/stream.go // meta:applied_seq, meta:rebuilt_at apply-loop bookkeeping // anything else service scratch KV (btmagnet:*, searchcov:*), /api/kv/value // // plus the tables claim_by_pid and local_file (localfile.go), and the // field index + FTS (search.go). // // Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is // exactly the merged records, so a record is the key range [file:/, // file:0), the field expression index covers only real fields, and every // change under `file:` is a real field change worth publishing on meta:events. const ( recordNS = "file:" rootsNS = "idx:roots/" ftsNS = "idx:fts/" dupNS = "dup:" streamNS = "ev:" hwmNS = "evhwm:" opTimeout = 30 * time.Second bulkBatch = 10_000 readerPage = 5_000 ) // MetaEventsStream receives a set/del entry for every merged-record change. It // is written by the cache engine in the same transaction as the change, and // served as SSE by /api/events/meta. const MetaEventsStream = "meta:events" // MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is // a contract requirement for the SSE wire — see docs/api-mediated-access.md // "Backing-store retention and lifecycle". const MetaEventsMaxLen = 100_000 // Client is meta-core's metadata store: the truth database, the cache, and // the apply loop between them. type Client struct { store *kv.Store // cache truth *kv.Store applier *applier mu sync.RWMutex keysMu sync.Mutex keysCount int64 keysAt time.Time } // NewClient creates an unconnected client. func NewClient() *Client { return &Client{} } // Connect opens (creating if needed) the truth database at truthPath and the // cache at cachePath. Call EnsureSchemaVersion, then Start. func (c *Client) Connect(truthPath, cachePath string) error { c.mu.Lock() defer c.mu.Unlock() truth, err := kv.Open(truthPath, kv.Options{ NoKVTable: true, Schema: truthSchema, WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64), ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), Readers: envInt("META_CORE_DB_READERS", 8), }) if err != nil { return fmt.Errorf("open truth %s: %w", truthPath, err) } store, err := kv.Open(cachePath, kv.Options{ EventStream: MetaEventsStream, EventMaxLen: MetaEventsMaxLen, EventKey: func(k string) bool { return strings.HasPrefix(k, recordNS) }, WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64), ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), Readers: envInt("META_CORE_DB_READERS", 8), Schema: searchSchema + cacheTablesSchema, OnChange: onChange, Synchronous: "OFF", }) if err != nil { truth.Close() return fmt.Errorf("open cache %s: %w", cachePath, err) } c.truth, c.store = truth, store log.Printf("[Storage] Opened truth %s, cache %s", truthPath, cachePath) return nil } // Start brings the cache level with truth (replay or rebuild) and starts the // apply loop. Writes before Start fail. func (c *Client) Start() error { applied, err := c.catchUp() if err != nil { return fmt.Errorf("cache catch-up: %w", err) } c.mu.Lock() c.applier = newApplier(c, applied) c.mu.Unlock() return nil } // Close stops the apply loop and closes both databases. func (c *Client) Close() error { c.mu.Lock() defer c.mu.Unlock() if c.applier != nil { c.applier.close() c.applier = nil } var errs []error if c.store != nil { errs = append(errs, c.store.Close()) c.store = nil } if c.truth != nil { errs = append(errs, c.truth.Close()) c.truth = nil } return errors.Join(errs...) } // IsConnected reports whether the databases are open. func (c *Client) IsConnected() bool { c.mu.RLock() defer c.mu.RUnlock() return c.store != nil && c.truth != nil } // Health checks that both databases answer. func (c *Client) Health() bool { s, err := c.kv() if err != nil { return false } t, err := c.truthStore() if err != nil { return false } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() return s.Ping(ctx) == nil && t.Ping(ctx) == nil } // Store exposes the cache engine (event streams for SSE and the schema // indexer). func (c *Client) Store() *kv.Store { c.mu.RLock() defer c.mu.RUnlock() return c.store } func (c *Client) kv() (*kv.Store, error) { c.mu.RLock() defer c.mu.RUnlock() if c.store == nil { return nil, fmt.Errorf("not connected") } return c.store, nil } func (c *Client) truthStore() (*kv.Store, error) { c.mu.RLock() defer c.mu.RUnlock() if c.truth == nil || c.applier == nil { return nil, fmt.Errorf("not connected") } return c.truth, nil } // update runs fn in one cache write transaction with the standard timeout. func (c *Client) update(fn func(tx *kv.Tx) error) error { s, err := c.kv() if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), opTimeout) defer cancel() return s.Update(ctx, fn) } func opCtx() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), opTimeout) } func recordPrefix(hashID string) string { return recordNS + hashID + "/" } func rootKey(hashID string) string { return rootsNS + hashID } func dupPrefix(hashID string) string { return dupNS + hashID + "/" } // isReservedFieldName reports whether field names bookkeeping rather than // metadata. `__fields__` was the Redis-era per-record field index; it no // longer exists, but a payload carrying it (an old snapshot) is still refused // so it can never become a phantom field. func isReservedFieldName(field string) bool { return field == "__fields__" } // splitRecordKey decomposes a key into (hashID, field) when it is a record // property key — `file:/`, split at the FIRST slash so CID-rooted // and UUID-rooted records parse alike. func splitRecordKey(key string) (hashID, field string, ok bool) { rest, found := strings.CutPrefix(key, recordNS) if !found { return "", "", false } hashID, field, found = strings.Cut(rest, "/") if !found || hashID == "" || field == "" || isReservedFieldName(field) { return "", "", false } return hashID, field, true } // isInternalKey reports keys that are derived bookkeeping, which the raw KV // editor must never write (it would desynchronise the derived state). func isInternalKey(key string) bool { for _, p := range []string{recordNS, CIDIndexPrefix, "idx:", dupNS, streamNS, hwmNS, "meta:", "meta-core:"} { if strings.HasPrefix(key, p) { return true } } return false } // SetServiceValue writes service scratch KV (btmagnet:*, searchcov:*) — the // only raw key writes left. Merged records and bookkeeping are refused. func (c *Client) SetServiceValue(key, value string) error { if key == "" || isInternalKey(key) { return fmt.Errorf("key %q is not a service key", key) } return c.update(func(tx *kv.Tx) error { return tx.Put(key, value) }) } // DeleteServiceValue deletes a service scratch key. Returns true if it existed. func (c *Client) DeleteServiceValue(key string) (bool, error) { if key == "" || isInternalKey(key) { return false, fmt.Errorf("key %q is not a service key", key) } existed := false err := c.update(func(tx *kv.Tx) error { var err error existed, err = tx.Delete(key) return err }) return existed, err } // ClearAllMetadata wipes BOTH databases: every claim, signer and group in // truth, and every derived row of the cache. Irreversible — truth holds votes // nothing else can recover. Returns the number of records that existed. func (c *Client) ClearAllMetadata() (int64, error) { truth, err := c.truthStore() if err != nil { return 0, err } roots, err := c.CountRoots() if err != nil { return 0, err } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() if err := truth.Update(ctx, func(tx *kv.Tx) error { for _, t := range []string{"claim", "principal", "cid_group"} { if _, err := tx.Exec(`DELETE FROM ` + t); err != nil { return err } } _, err := tx.Exec(`DELETE FROM meta WHERE name = ?`, lastSeqName) return err }); err != nil { return 0, fmt.Errorf("clear truth: %w", err) } for _, prefix := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS} { for { var n int64 if err := c.update(func(tx *kv.Tx) error { var err error n, err = tx.DeletePrefix(prefix, bulkBatch) return err }); err != nil { return 0, fmt.Errorf("clear %s: %w", prefix, err) } if n < bulkBatch { break } } } if err := c.update(func(tx *kv.Tx) error { for _, q := range []string{`DELETE FROM fts`, `DELETE FROM claim_by_pid`, `DELETE FROM local_file`} { if _, err := tx.Exec(q); err != nil { return err } } return tx.Put(appliedSeqKey, "0") }); err != nil { return 0, fmt.Errorf("clear cache: %w", err) } c.applier.mu.Lock() c.applier.applied = 0 c.applier.mu.Unlock() log.Printf("[Storage] Cleared both databases (%d records)", roots) return roots, nil } // Derived counters, maintained by onChange in the same transaction as the // change they count (kv.Tx.AddCounter): the root count behind search's `total` // and /api/stats, and the per-field schema behind /api/schema. const ( rootsCountKey = "idx:count/roots" countersReady = "idx:count/ready" counterRebuild = 5000 ) // onChange keeps the derived counters exact. It sees only real changes (kv // skips unchanged Puts) with the old and new values, so a modified field moves // one count from its old label to its new one. func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) { if strings.HasPrefix(k, rootsNS) { switch { case hasNew && !hadOld: tx.AddCounter(rootsCountKey, 1) case hadOld && !hasNew: tx.AddCounter(rootsCountKey, -1) } return } if !strings.HasPrefix(k, recordNS) { return } if hadOld { if ck, ok := schema.CounterKey(k, old); ok { tx.AddCounter(ck, -1) } } if hasNew { if ck, ok := schema.CounterKey(k, v); ok { tx.AddCounter(ck, 1) } } } // EnsureCounters builds the derived counters when they have never been built — // a store created before they existed, or one restored from outside meta-core. // Blocking and single-transaction (writes wait for it; ~1–2 min at millions of // keys, once), because a build racing live deltas would double-count. func (c *Client) EnsureCounters() (bool, error) { s, err := c.kv() if err != nil { return false, err } ctx, cancel := opCtx() _, ready, err := s.Get(ctx, countersReady) cancel() if err != nil || ready { return false, err } return true, c.RebuildCounters() } // RebuildCounters recomputes every derived counter from the records, in one // transaction (writers wait for it). func (c *Client) RebuildCounters() error { s, err := c.kv() if err != nil { return err } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() return s.Update(ctx, func(tx *kv.Tx) error { for _, p := range []string{schema.CounterPrefix, rootsCountKey} { if _, err := tx.DeletePrefix(p, 0); err != nil { return err } } roots, err := tx.Count(tx.Context(), rootsNS) if err != nil { return err } tx.AddCounter(rootsCountKey, roots) after := "" for { rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild) if err != nil { return err } for _, row := range rows { if ck, ok := schema.CounterKey(row.K, row.V); ok { tx.AddCounter(ck, 1) } } if len(rows) < counterRebuild { break } after = rows[len(rows)-1].K } return tx.Put(countersReady, "1") }) } // readRecord returns a record's flat fields (nil when it has none). func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) { prefix := recordPrefix(hashID) rows, err := r.Range(ctx, prefix, "", 0) if err != nil { return nil, fmt.Errorf("read %s: %w", hashID, err) } if len(rows) == 0 { return nil, nil } out := make(map[string]string, len(rows)) for _, row := range rows { out[row.K[len(prefix):]] = row.V } return out, nil } // GetMetadataFlat retrieves all metadata for a file as a flat map. func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) { s, err := c.kv() if err != nil { return nil, err } ctx, cancel := opCtx() defer cancel() return readRecord(ctx, s, hashID) } // GetAllHashIDs returns every root id, sorted. func (c *Client) GetAllHashIDs() ([]string, error) { s, err := c.kv() if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout) defer cancel() var out []string err = s.ScanPrefix(ctx, rootsNS, readerPage, func(rows []kv.KV) error { for _, row := range rows { out = append(out, row.K[len(rootsNS):]) } return nil }) return out, err } // GetProperty retrieves a single property value ("" when absent). func (c *Client) GetProperty(hashID, property string) (string, error) { s, err := c.kv() if err != nil { return "", err } ctx, cancel := opCtx() defer cancel() v, _, err := s.Get(ctx, recordPrefix(hashID)+property) return v, err } // CountFiles returns the number of indexed roots. func (c *Client) CountFiles() (int, error) { n, err := c.CountRoots() return int(n), err } // LookupPathByCID resolves a bare CID to its file path on disk via the // reverse index: one lookup followed by a single read of file:/filePath. // // Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS // CID, a btih, etc. (they all alias to the same root). The bare CID is what // the editor passes to /api/file/{cid}/info and what plugins write (e.g. // `backdrop=`). // // Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is // resolved through this same path: the plugin output directory is a // watcher root (see config.DefaultWatcherPaths), so every plugin artefact // gets a midhash alias and resolves through the reverse index like any // other file. There is no path-companion sidecar field. func (c *Client) LookupPathByCID(cidStr string) (string, error) { uuid, err := c.GetByCID(cidStr) if err != nil { return "", err } if uuid == "" { return "", nil } fp, err := c.GetProperty(uuid, "filePath") if err != nil { return "", err } return normalizeFilesRelativePath(fp), nil } // normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can // safely filepath.Join(FilesPath, …) without double-prefixing. Different // plugins store paths inconsistently — poster/backdrop use FilesPath-relative // ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…"). func normalizeFilesRelativePath(p string) string { return strings.TrimPrefix(p, "/files") } // DeleteProperty deletes a single property. // // GetMemoryInfo reports the store's footprint for dashboards: the database // file plus its WAL, human-readable. (Under Redis this was used_memory_human; // the data no longer lives in RAM, so the disk footprint is the honest figure.) func (c *Client) GetMemoryInfo() (string, error) { if _, err := c.kv(); err != nil { return "", err } return humanBytes(c.DBSize()), nil } // DBSize is both database files plus their WALs, in bytes. func (c *Client) DBSize() int64 { c.mu.RLock() defer c.mu.RUnlock() var n int64 for _, st := range []*kv.Store{c.store, c.truth} { if st != nil { n += st.FileSize() } } return n } func humanBytes(n int64) string { const unit = 1024 if n < unit { return fmt.Sprintf("%dB", n) } div, exp := int64(unit), 0 for m := n / unit; m >= unit; m /= unit { div *= unit exp++ } return fmt.Sprintf("%.2f%c", float64(n)/float64(div), "KMGTPE"[exp]) } // ClearAllMetadata deletes all file metadata, the root index, the CID reverse // index, duplicates and search rows. Returns the number of roots deleted. // // The CID reverse index MUST be purged too: ResolveRoot consults it, so a // surviving cid: entry pointing at a root we just deleted silently // redirects the next /meta/ write back into that dead root, resurrecting // a zombie record under an old identity. // // ScanKeys lists the immediate children at the given prefix, S3-style. // // When delimiter is set, keys with a delimiter occurrence after the prefix // collapse into a branch path (the prefix up to and including that first // delimiter); the rest are returned as leaves. The walk jumps over each // branch's whole key range once seen, so listing `file:` costs one read per // record, not one per key. Stops when leaves+branches reaches maxResults; the // returned cursor (the next key to examine, "0" when exhausted) continues it. func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ([]string, []string, string, error) { s, err := c.kv() if err != nil { return nil, nil, "", err } if maxResults <= 0 { maxResults = 500 } from := prefix if cursorStr != "" && cursorStr != "0" && cursorStr > prefix { from = cursorStr } end := kv.PrefixEnd(prefix) ctx, cancel := opCtx() defer cancel() var leaves, branches []string next := "0" op := ">=" // after a leaf the bound becomes exclusive (k > leaf) for { query := `SELECT k FROM kv WHERE k ` + op + ` ?` args := []any{from} if end != "" { query += ` AND k < ?` args = append(args, end) } var k string err := s.DB().QueryRowContext(ctx, query+` ORDER BY k LIMIT 1`, args...).Scan(&k) if errors.Is(err, sql.ErrNoRows) { break } if err != nil { return nil, nil, "", err } if len(leaves)+len(branches) >= maxResults { next = k break } suffix := k[len(prefix):] if delimiter != "" { if idx := strings.Index(suffix, delimiter); idx >= 0 { branch := prefix + suffix[:idx+len(delimiter)] branches = append(branches, branch) if from = kv.PrefixEnd(branch); from == "" { break } op = ">=" continue } } leaves = append(leaves, k) from, op = k, ">" } leaves, branches = serviceOnly(leaves), serviceOnly(branches) sort.Strings(leaves) sort.Strings(branches) return leaves, branches, next, nil } // serviceOnly drops merged-record and bookkeeping keys: the /api/kv browser // shows service scratch KV only. func serviceOnly(keys []string) []string { out := keys[:0] for _, k := range keys { if !isInternalKey(k) { out = append(out, k) } } return out } // FindMatch is a single value-search hit. type FindMatch struct { Key string `json:"key"` Value string `json:"value"` EntryPath string `json:"entryPath"` Field string `json:"field"` } // FindByValue returns keys whose path ends with / (for each field in // fields) and whose value contains the substring, case-insensitively. // truncated is true when limit was hit. // // EntryPath is the parent prefix up to (and including) the slash before // field — i.e. the file branch path. Useful for revealing the matched entry // in the tree. func (c *Client) FindByValue(contains string, fields []string, limit int) ([]FindMatch, bool, error) { s, err := c.kv() if err != nil { return nil, false, err } if contains == "" || len(fields) == 0 { return nil, false, nil } if limit <= 0 { limit = 200 } ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout) defer cancel() needle := strings.ToLower(contains) var matches []FindMatch truncated := false for _, field := range fields { field = strings.TrimSpace(field) if field == "" || isReservedFieldName(field) { continue } if len(matches) >= limit { truncated = true break } suffix := "/" + field // Suffix match over every key (records and non-record keys alike, as // the SCAN "*/" it replaces did). Value containment is checked in // Go: SQLite's lower() folds ASCII only. rows, err := s.DB().QueryContext(ctx, `SELECT k, v FROM kv WHERE substr(k, -?) = ? AND instr(lower(v), ?) > 0 ORDER BY k`, len(suffix), suffix, asciiLower(needle)) if err != nil { return nil, false, err } for rows.Next() { var k, v string if err := rows.Scan(&k, &v); err != nil { rows.Close() return nil, false, err } if isInternalKey(k) || !strings.Contains(strings.ToLower(v), needle) { continue } matches = append(matches, FindMatch{Key: k, Value: v, EntryPath: strings.TrimSuffix(k, field), Field: field}) if len(matches) >= limit { truncated = true break } } rows.Close() } sort.Slice(matches, func(i, j int) bool { return matches[i].Key < matches[j].Key }) return matches, truncated, nil } // SearchKeys returns up to limit keys whose full path contains the substring, // in key order. truncated is true when the search stopped at limit. func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) { s, err := c.kv() if err != nil { return nil, false, err } if contains == "" { return nil, false, nil } if limit <= 0 { limit = 500 } ctx, cancel := opCtx() defer cancel() rows, err := s.DB().QueryContext(ctx, `SELECT k FROM kv WHERE instr(k, ?) > 0 ORDER BY k LIMIT ?`, contains, limit+1) if err != nil { return nil, false, err } defer rows.Close() results := make([]string, 0, limit) for rows.Next() { var k string if err := rows.Scan(&k); err != nil { return nil, false, err } results = append(results, k) } if err := rows.Err(); err != nil { return nil, false, err } truncated := len(results) > limit if truncated { results = results[:limit] } return serviceOnly(results), truncated, nil } // GetKeyInfo returns the key's type — "string" when it exists (every row is a // string now), with its value. exists is false when the key is absent. func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, err error) { s, err := c.kv() if err != nil { return "", "", false, err } if isInternalKey(key) { return "", "", false, nil } ctx, cancel := opCtx() defer cancel() v, ok, err := s.Get(ctx, key) if err != nil || !ok { return "", "", false, err } return "string", v, true, nil } // GetPrefix returns the key namespace prefix. Always empty: meta-core owns its // database file outright, so there is nothing to share a keyspace with. func (c *Client) GetPrefix() string { return "" } // XAdd appends an event to a stream. maxLen > 0 trims to about that many // entries; maxLen <= 0 keeps everything. func (c *Client) XAdd(stream string, maxLen int64, fields map[string]interface{}) (string, error) { values := make(map[string]string, len(fields)) for k, v := range fields { values[k] = fmt.Sprint(v) } var id string err := c.update(func(tx *kv.Tx) error { var err error id, err = tx.Append(stream, maxLen, values) return err }) return id, err } // AppendEvents appends a batch of entries to a stream in one transaction. func (c *Client) AppendEvents(stream string, maxLen int64, entries []map[string]string) error { if len(entries) == 0 { return nil } return c.update(func(tx *kv.Tx) error { for _, e := range entries { if _, err := tx.Append(stream, 0, e); err != nil { return err } } if maxLen > 0 { return tx.TrimAll(stream, maxLen) } return nil }) } // ClearStream deletes every entry of a stream (its ID high-water mark // survives, so IDs keep increasing across the clear). func (c *Client) ClearStream(stream string) error { return c.update(func(tx *kv.Tx) error { return tx.ClearStream(stream) }) } // StreamEntry is one stream entry. type StreamEntry struct { ID string Values map[string]interface{} } // XRevRange reads stream entries from newest to oldest. start must be "+" or // empty (newest); stop is "-"/empty (oldest) or "(" for an exclusive lower // bound — the only shapes callers use. count > 0 caps the result. func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) { s, err := c.kv() if err != nil { return nil, err } if start != "" && start != "+" { return nil, fmt.Errorf("XRevRange: unsupported start %q", start) } after := "" switch { case stop == "" || stop == "-": case strings.HasPrefix(stop, "("): after = stop[1:] default: return nil, fmt.Errorf("XRevRange: unsupported stop %q", stop) } ctx, cancel := opCtx() defer cancel() entries, err := s.RevRange(ctx, stream, after, int(count)) if err != nil { return nil, err } out := make([]StreamEntry, 0, len(entries)) for _, e := range entries { values := make(map[string]interface{}, len(e.Values)) for k, v := range e.Values { values[k] = v } out = append(out, StreamEntry{ID: e.ID, Values: values}) } return out, nil } // GetStreamLength returns the number of surviving entries of a stream. func (c *Client) GetStreamLength(stream string) (int64, error) { s, err := c.kv() if err != nil { return 0, err } ctx, cancel := opCtx() defer cancel() return s.Len(ctx, stream) }