metastar/meta-core
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
Details

Browse Source
This commit is contained in:
Worph
2026-10-19 20:51:00 +00:00
parent c53146e514
commit dc5cbdfb9b
80 changed files with 4601 additions and 6077 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/admin_migrate.go
-131
View File
@@ -1,131 +0,0 @@
package api
import (
"net/http"
"strconv"
)
// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
// stranded midhash-rooted entries into their UUID roots. See
// storage.MigrateDualRoots for the data-flow detail and
// docs/api-mediated-access.md for the bug background.
//
// Returns {fixed: N} on success. Safe to call repeatedly — entries already
// migrated are skipped on the next pass.
func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.MigrateDualRootsWithTimeout()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the
// retired `domain=film|tv` values to `screen` and backfills `workForm` from
// `contentKind` (METADATA_KEYS.md §14.17). See
// storage.MigrateDomainScreen for why the vocabulary change is paid in the
// data rather than with a read-side alias.
//
// Returns {fixed: N} — records changed, not records examined. Safe to call
// repeatedly: a record already on the current vocabulary is skipped.
func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.MigrateDomainScreenWithTimeout()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
// `anilistid` on a site card, and `workcid` / position keys on any card
// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
// why chapters and anchor cards' ids are deliberately left alone.
//
// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
// keys, so a bare call only reports what it would delete. An unparseable
// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
// sweep that ran.
//
// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
// records, byKey counts fields per key. Safe to call repeatedly: a swept
// record plans nothing on the next pass.
func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
apply := false
if v := r.URL.Query().Get("apply"); v != "" {
parsed, err := strconv.ParseBool(v)
if err != nil {
writeError(w, http.StatusBadRequest, "apply must be true or false")
return
}
apply = parsed
}
res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"dryRun": !apply,
"planned": res.Planned,
"fixed": res.Fixed,
"byKey": res.ByKey,
})
}
// handleMigrateUserCids triggers the one-shot sweep that builds
// `udl:idx:user:<uid>:cids` for User Data Layer records written before that
// index existed. See storage.UDLBackfillUserCids.
//
// The index is what makes whole-profile export a set read instead of a
// keyspace scan, and it is maintained by the upsert script from now on — so
// this only ever matters once, for data already on disk. Until it has run, a
// profile that predates the index exports as **empty**, which means a
// cross-instance recovery reports success having restored nothing.
//
// Returns {fixed: N} — cids added, not cids examined. Safe to call repeatedly:
// a second pass adds nothing.
func (s *Server) handleMigrateUserCids(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.UDLBackfillUserCids()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
internal/api/admin_sweep_test.go
-93
View File
@@ -1,93 +0,0 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
// newSweepTestServer is a Server over miniredis with one literature card
// carrying a copied `workcid`.
func newSweepTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
for k, v := range map[string]string{
"domain": "literature",
"fileType": "card",
"workcid": "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
} {
if err := c.SetProperty("card-1", k, v); err != nil {
t.Fatalf("seed %s: %v", k, err)
}
}
return &Server{storage: c}
}
type sweepResponse struct {
Success bool `json:"success"`
DryRun bool `json:"dryRun"`
Planned int `json:"planned"`
Fixed int `json:"fixed"`
ByKey map[string]int `json:"byKey"`
}
func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
t.Helper()
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
s.handleSweepLiteratureEchoes(rr, req)
var body sweepResponse
if rr.Code == http.StatusOK {
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
t.Fatalf("decode %s: %v", rr.Body.String(), err)
}
}
return rr, body
}
// A bare call must never delete: the default is the dry run.
func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
s := newSweepTestServer(t)
rr, body := postSweep(t, s, "")
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
}
if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
t.Errorf("dry run = %+v", body)
}
if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
t.Errorf("dry run deleted workcid")
}
rr, body = postSweep(t, s, "?apply=true")
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
}
if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
t.Errorf("apply = %+v", body)
}
if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
t.Errorf("apply left workcid = %q", v)
}
}
// A typo must not pass for a sweep that ran.
func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
s := newSweepTestServer(t)
rr, _ := postSweep(t, s, "?apply=yes-please")
if rr.Code != http.StatusBadRequest {
t.Errorf("status = %d, want 400", rr.Code)
}
}
internal/api/files_register_test.go
+16 -25
View File
@@ -9,24 +9,16 @@ import (
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/storage"
)
// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
// newRegisterTestServer wires handleRegisterFile to a temp store and a temp
// FILES_PATH. Built directly, not through NewServer, so no watcher runs —
// which is exactly the configuration this route exists for.
func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
func newRegisterTestServer(t *testing.T) (*Server, string) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
files := t.TempDir()
return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
return &Server{storage: newTestStorage(t), config: &config.Config{FilesPath: files}}, files
}
func writeFixture(t *testing.T, files, rel, body string) string {
@@ -52,20 +44,20 @@ func postRegister(t *testing.T, s *Server, path string) (int, registerFileRespon
return rr.Code, resp
}
func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
func streamLen(t *testing.T, s *Server) int {
t.Helper()
entries, err := mr.Stream("file:events")
n, err := s.storage.GetStreamLength("file:events")
if err != nil {
return 0
t.Fatalf("stream length: %v", err)
}
return len(entries)
return int(n)
}
// The whole point of the route: with the watcher off, a file written under
// plugin/ becomes a record resolvable by its midhash, and meta-share hears
// about it on file:events.
func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
s, files := newRegisterTestServer(t)
rel := "plugin/subtitles/ab12cd34.eng.srt"
full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
@@ -85,20 +77,19 @@ func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
t.Fatalf("filePath = %q, want %q", fp, full)
}
entries, err := mr.Stream("file:events")
entries, err := s.storage.XRevRange("file:events", "+", "-", 0)
if err != nil || len(entries) != 1 {
t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
}
vals := strings.Join(entries[0].Values, "|")
if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
if vals := entries[0].Values; vals["midhash256"] != resp.CID || vals["path"] != rel {
t.Fatalf("event must carry the midhash and the relative path, got %v", vals)
}
}
// Re-uploading the same subtitle is a no-op: same root, no second event (which
// would make meta-share re-chunk and re-provide the file).
func TestRegisterFile_IsIdempotent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
s, files := newRegisterTestServer(t)
rel := "plugin/subtitles/same.eng.srt"
writeFixture(t, files, rel, "same bytes")
@@ -110,7 +101,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
if second.Root != first.Root || second.CID != first.CID {
t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
}
if n := streamLen(t, mr); n != 1 {
if n := streamLen(t, s); n != 1 {
t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
}
}
@@ -118,7 +109,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
// Only plugin output may be registered here. /files/watch is the user's
// library and belongs to the watcher; traversal must not escape either.
func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
s, files := newRegisterTestServer(t)
writeFixture(t, files, "watch/movie.mkv", "x")
for _, p := range []string{
"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
@@ -131,7 +122,7 @@ func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
}
func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
s, files := newRegisterTestServer(t)
if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
t.Errorf("missing file: want 404, got %d", code)
}
internal/api/files_tuples_test.go
+2 -8
View File
@@ -7,18 +7,12 @@ import (
"sort"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
c := newTestStorage(t)
return &Server{storage: c}, c
}
@@ -105,7 +99,7 @@ func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
}
func TestFileTuples_StorageNotConnected(t *testing.T) {
s := &Server{storage: storage.NewClient("")}
s := &Server{storage: storage.NewClient()}
code, _ := getFileTuples(t, s, "")
if code != http.StatusServiceUnavailable {
internal/api/identity_gate_test.go
+2 -9
View File
@@ -1,6 +1,6 @@
package api
// End-to-end exercise of the account gate against a live router + miniredis:
// End-to-end exercise of the account gate against a live router + a temp store:
// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
// both the key and every row are gone. Deliberately drives the real HTTP
// handlers rather than the packages beneath them, because the bugs that matter
@@ -16,21 +16,14 @@ import (
"net/http/httptest"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
func newTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
stor := storage.NewClient("")
if err := stor.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = stor.Close() })
stor := newTestStorage(t)
s := &Server{
config: &config.Config{MetaCorePath: t.TempDir()},
internal/api/kv.go
+3 -3
View File
@@ -206,10 +206,10 @@ type KVSearchResponse struct {
// KVFindResponse is the response for GET /api/kv/find
type KVFindResponse struct {
Contains string `json:"contains"`
Fields []string `json:"fields"`
Contains string `json:"contains"`
Fields []string `json:"fields"`
Matches []storage.FindMatch `json:"matches"`
Truncated bool `json:"truncated"`
Truncated bool `json:"truncated"`
}
// KVValueResponse is the response for GET /api/kv/value
internal/api/metadata.go
+5 -6
View File
@@ -40,9 +40,9 @@ type MetadataSearchRequest struct {
Limit int `json:"limit,omitempty"`
// Filters is the structured form: AND across entries, OR within an
// entry's Values. Evaluated in memory against the search index alongside
// the free-text Query, so an id-anchored lookup costs a map probe per
// record rather than a Redis round-trip per record.
// entry's Values. Narrowed in SQL through the field index alongside the
// free-text Query, then verified per record (see serveIndexedSearch), so an
// id-anchored lookup costs an index probe rather than a read per record.
//
// Property/PropertyValue is the single-filter legacy shape and folds into
// this list (see normalizedFilters) with Exact=false, preserving its
@@ -445,9 +445,8 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
}
// Everything else — free text, structured filters, both, or neither
// (which returns the first `limit`) — is served from the in-memory search
// index (search_index.go). Matching and the returned metadata both come
// from memory, so a query costs zero Redis round-trips.
// (which returns the first `limit`) — is served from the store's derived
// search indexes (search_index.go → storage.Search).
//
// There is deliberately no second path. The property branch used to fall
// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which
internal/api/schema_handlers.go
+10 -13
View File
@@ -3,10 +3,10 @@ package api
import (
"encoding/json"
"net/http"
"time"
)
// handleSchemaGet returns the live per-field schema derived from meta:events.
// handleSchemaGet returns the per-field schema (recomputed from the store every
// few minutes; see schema.Indexer).
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
@@ -17,29 +17,26 @@ func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(snap)
}
// handleSchemaRescan re-emits a set event for every existing property via the
// meta publisher, waits for the indexer to drain, and returns the refreshed
// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
// handleSchemaRescan recomputes the schema from a full scan of the store and
// returns it. It no longer republishes every property into meta:events — the
// indexer reads the store itself — so a rescan floods no consumer.
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
count, err := s.RepublishMetadata()
snap, err := s.schemaIndexer.Rebuild(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.schemaIndexer.WaitForDrain(10 * time.Second)
snap := s.schemaIndexer.Snapshot()
snap.Source = "rescan"
out := *snap
out.Source = "rescan"
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"published_events": count,
"schema": snap,
"published_events": 0,
"schema": &out,
})
}
internal/api/search_index.go
+33 -195
View File
@@ -1,221 +1,59 @@
package api
import (
"log"
"net/http"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/metazla/meta-core/internal/storage"
)
// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
// from Redis in the background. Records ingest continuously, so a newly stored
// record becomes searchable within one interval; the rebuild is a single
// batched bulk read (storage.GetSearchIndexTuples), off the request path.
const searchIndexRefreshInterval = 60 * time.Second
// searchIndexReconcileEvery is how many refresh ticks pass between full
// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full
// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all.
const searchIndexReconcileEvery = 30
// searchIndex is an in-memory snapshot of every record's search haystack
// (hashID + lowercased searchable fields), kept current from Redis. It exists
// so POST /api/metadata/search matches in memory (string ops over a slice)
// instead of doing a SCAN+MGET Redis round-trip per record — the latter took
// 40s+ over ~72k records and made a live parallel search path unusable.
// serveIndexedSearch answers a search from the database's derived indexes
// (storage.Search): SQL narrows to a superset of candidates through the field
// expression index and the FTS5 trigram table, then every candidate is checked
// here with the exact matchers below — a token-AND match against the record's
// haystack (or a whole-query hashID substring, preserving the legacy
// behaviour) AND every structured filter.
//
// # Two update paths, and why
//
// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a
// complete fresh copy. That is O(corpus) regardless of how much changed, and it
// used to run every 60 s forever. Measured on a 19 622-record staging corpus
// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side
// held 783 MB for data Redis stores in 135 MB — largely the double copy.
//
// `refreshIncremental` re-reads only the records that changed, tracked by
// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady
// state; the full rebuild runs once at startup (there is nothing to be
// incremental against) and then only as an explicit repair.
//
// The distinction matters because meta-core is the platform's permanent,
// additive store — every search hit on every gateway is retained — so a rebuild
// priced against the whole corpus is a cost that only ever grows.
type searchIndex struct {
mu sync.RWMutex
// byID is the authoritative content; tuples is a stable-order view of it
// rebuilt on each update. Map iteration order in Go is randomised, so
// serving a limited query straight off the map would return a different
// subset each call — the slice is what keeps results stable.
byID map[string]storage.SearchTuple
tuples []storage.SearchTuple
ready atomic.Bool
}
// snapshot returns the current tuples slice and whether the index has built at
// least once. Every update swaps in a fresh slice under the write lock (never
// mutates in place), so the returned slice is safe to read lock-free.
func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.tuples, idx.ready.Load()
}
// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds
// the write lock.
func (idx *searchIndex) rebuildViewLocked() {
view := make([]storage.SearchTuple, 0, len(idx.byID))
for _, t := range idx.byID {
view = append(view, t)
}
// Stable across rebuilds so a limited query returns a consistent page
// rather than a fresh random subset every interval.
sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID })
idx.tuples = view
}
// refresh rebuilds the whole index from Redis (a batched bulk read). Used for
// the initial build and as an explicit repair — not the steady state.
func (idx *searchIndex) refresh(stor *storage.Client) {
if stor == nil || !stor.IsConnected() {
return
}
tuples, err := stor.GetSearchIndexTuples()
if err != nil {
log.Printf("[API] search index refresh failed: %v", err)
return
}
byID := make(map[string]storage.SearchTuple, len(tuples))
for _, t := range tuples {
byID[t.HashID] = t
}
idx.mu.Lock()
idx.byID = byID
idx.rebuildViewLocked()
idx.mu.Unlock()
idx.ready.Store(true)
}
// refreshIncremental drains the dirty set and patches only those records.
// Returns whether more dirty entries remain, so a burst is caught up over
// consecutive passes rather than one per interval.
//
// A record in the dirty set but absent from storage was deleted and is evicted
// here — otherwise it would keep answering searches until the next full
// rebuild. A batch that fails after the (destructive) drain is re-marked by
// storage so it is retried rather than silently lost.
func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) {
if stor == nil || !stor.IsConnected() {
return false
}
ids, more, err := stor.DrainDirty()
if err != nil {
log.Printf("[API] search index drain failed: %v", err)
return false
}
if len(ids) == 0 {
return false
}
tuples, gone, err := stor.GetSearchIndexTuplesFor(ids)
if err != nil {
stor.MarkDirty(ids...)
log.Printf("[API] incremental search index refresh failed: %v", err)
return false
}
idx.mu.Lock()
if idx.byID == nil {
idx.byID = make(map[string]storage.SearchTuple, len(tuples))
}
for _, t := range tuples {
idx.byID[t.HashID] = t
}
for _, id := range gone {
delete(idx.byID, id)
}
idx.rebuildViewLocked()
idx.mu.Unlock()
log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples))
return more
}
// runRefreshLoop builds the index once in full, then keeps it current from the
// dirty set, with a periodic full reconcile as a safety net.
//
// The reconcile exists because a *missed* dirty mark is silent: no error, just
// a record whose indexed copy quietly diverges from storage until something
// forces a rebuild. Marking is spread across eight write paths plus anything
// added later, and the cost of getting it wrong is invisible — so the index
// should not depend on that coverage being perfect forever. Every
// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs
// whatever drifted.
//
// That still drops the full rebuild from every 60 s to every 30 min — a ~30×
// reduction in the steady-state cost that measured at ~45 % of all Redis CPU —
// while keeping the self-healing property that made the naive loop safe.
func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
idx.refresh(stor)
ticker := time.NewTicker(searchIndexRefreshInterval)
defer ticker.Stop()
ticks := 0
for range ticker.C {
ticks++
if ticks%searchIndexReconcileEvery == 0 {
idx.refresh(stor)
continue
}
// Drain until caught up, so a burst larger than one batch does not take
// one interval per batch to land.
for idx.refreshIncremental(stor) {
}
}
}
// serveIndexedSearch answers a search from the in-memory index: a token-AND
// match against each record's haystack (or a whole-query hashID substring,
// preserving the legacy behaviour) AND every structured filter, then returns
// the full metadata for the <=limit survivors. No Redis round-trip per scanned
// record — matching and results both come from memory.
// Until 2026-10 this ran against an in-memory copy of every record, rebuilt
// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh)
// and is gone. The matchers did not change, so neither did the semantics.
//
// This is the ONLY search path. Free text, filters, both, or neither all land
// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
tuples, ready := s.search.snapshot()
if !ready {
// First call before the background build finished: warm inline with the
// efficient batched read (seconds), never the per-record SCAN fallback.
s.search.refresh(s.storage)
tuples, _ = s.search.snapshot()
}
tokens := strings.Fields(strings.ToLower(req.Query))
queryLower := strings.ToLower(req.Query)
filters := normalizedFilters(req)
results := make([]MetadataSearchResult, 0, req.Limit)
for _, t := range tuples {
if len(results) >= req.Limit {
break
}
narrow := make([]storage.SearchFilter, 0, len(filters))
for _, f := range filters {
narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
}
hits, err := s.storage.Search(req.Query, narrow, req.Limit, func(hashID string, fields map[string]string) bool {
if req.Query != "" &&
!matchAllTokens(t.Haystack, tokens) &&
!strings.Contains(strings.ToLower(t.HashID), queryLower) {
continue
!matchAllTokens(storage.Haystack(fields), tokens) &&
!strings.Contains(strings.ToLower(hashID), queryLower) {
return false
}
if !recordMatchesFilters(t.Fields, filters) {
continue
}
// Full metadata is cached in the index — return it straight from memory,
// no Redis round-trip per result.
results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
return recordMatchesFilters(fields, filters)
})
if err != nil {
writeError(w, http.StatusInternalServerError, "search failed: "+err.Error())
return
}
total, err := s.storage.CountRoots()
if err != nil {
total = 0
}
results := make([]MetadataSearchResult, 0, len(hits))
for _, h := range hits {
results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields})
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: len(tuples),
Total: int(total),
})
}
internal/api/server.go
+20 -47
View File
@@ -39,7 +39,6 @@ type Server struct {
metaPublisher *events.MetaPublisher
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler
search *searchIndex
// challenges backs proof-of-possession on reveal/delete. In-process, so a
// restart invalidates the ones in flight — see identity/challenge.go.
challenges *identity.ChallengeStore
@@ -51,7 +50,7 @@ type Server struct {
router *mux.Router
server *http.Server
// stats caches the keyspace-walk half of GET /api/stats so a polling
// dashboard never puts a multi-second Redis walk on the request path.
// dashboard never puts a multi-second store walk on the request path.
stats *statsCache
}
@@ -71,7 +70,6 @@ func NewServer(
config: cfg,
leaderProvider: leaderProvider,
storage: stor,
search: &searchIndex{},
router: mux.NewRouter(),
stats: &statsCache{},
}
@@ -80,7 +78,7 @@ func NewServer(
var scanFunc mounts.ScanFunc
if cfg.EnableFileWatcher {
s.watcherDispatcher = watcher.NewDispatcher()
// Wire storage client for Redis Stream publishing
// Wire storage client for file:events publishing
if stor != nil {
s.watcherDispatcher.SetStorageClient(stor)
}
@@ -90,7 +88,7 @@ func NewServer(
} else {
s.fileWatcher = fileWatcher
// Wire storage so the watcher can persist (path,size,mtime,cid)
// tuples to Redis as it computes midhashes.
// tuples to the store as it computes midhashes.
if stor != nil {
fileWatcher.SetStorageClient(stor)
}
@@ -112,7 +110,7 @@ func NewServer(
// Pass republish callback to handlers (handles nil metaPublisher case)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
// Hydrate the in-memory state registry from Redis tuples so
// Hydrate the in-memory state registry from stored tuples so
// the first scan tick after boot can skip ComputeMidHash256
// for files whose (size, mtime) match.
configs := watchersManager.List()
@@ -123,7 +121,7 @@ func NewServer(
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
} else {
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
}
}
}
@@ -272,20 +270,8 @@ func (s *Server) setupRoutes() {
// without going through Authelia.
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
// Admin: reunify stranded midhash-rooted entries into their UUID roots
// (the dual-root pattern caused by historical writes that bypassed CID
// resolution).
s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
// (METADATA_KEYS.md §14.17). Idempotent.
s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST")
// One-shot sweep of query echoes on literature cards (meta-read
// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
// SSE event streams. Mediated mirror of the Redis Streams; consumers
// reach them over HTTP instead of touching Redis directly. Auth-bypass
// SSE event streams. The only way consumers see meta-core's change
// streams — they never touch the database. Auth-bypass
// applies but exposure should be inside-only (do NOT proxy through
// Caddy publicly). See docs/api-mediated-access.md "Auth".
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
@@ -363,44 +349,31 @@ func (s *Server) Start() error {
s.mountStatsPoller.Start()
}
// Build + periodically refresh the in-memory search index so
// POST /api/metadata/search matches in memory instead of a Redis
// round-trip per record (see search_index.go).
if s.storage != nil {
go s.search.runRefreshLoop(s.storage)
}
// Start meta publisher (if storage is connected)
// Publishes keyspace notifications to meta:events stream
// Start meta publisher (if storage is connected). Live change events are
// written by the storage engine itself; the publisher owns the bulk
// republish below.
if s.storage != nil && s.storage.IsConnected() {
s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient())
s.metaPublisher = events.NewMetaPublisher(s.storage)
if err := s.metaPublisher.Start(); err != nil {
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
} else {
// Schema indexer consumes the same stream to derive a live
// per-field schema (type/hint/breakdown). Start it before the
// republish flood so it captures the bootstrap events.
s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
// Schema indexer: per-field schema (type/hint/breakdown) from a
// periodic scan of the store.
s.schemaIndexer = schema.NewIndexer(s.storage.Store())
if err := s.schemaIndexer.Start(); err != nil {
log.Printf("[API] Warning: failed to start schema indexer: %v", err)
s.schemaIndexer = nil
}
// Republish events for all existing metadata
// Republish events for all existing metadata. Consumers that
// rebuild by replaying meta:events from the beginning (meta-fuse)
// depend on it.
go func() {
// Build any missing per-record field indexes first, in ONE
// keyspace pass. The republish below reads every record, and
// an un-indexed record read costs a full-keyspace SCAN — on a
// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
// paid again on every restart). Warming first turns the whole
// republish into index-served reads. Idempotent, so a
// fully-indexed store just pays the single scan.
// See storage/field_index.go.
started := time.Now()
if n, err := s.storage.WarmFieldIndexes(); err != nil {
log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
log.Printf("[API] Warning: search row rebuild failed: %v", err)
} else if n > 0 {
log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
}
count, err := s.RepublishMetadata()
internal/api/snapshot.go
+1 -1
View File
@@ -103,7 +103,7 @@ func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, result)
}
// handleSnapshotWipe clears Redis metadata.
// handleSnapshotWipe clears all stored metadata.
// POST /api/snapshot/wipe?scope=metadata
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
internal/api/sse_events.go
+72 -87
View File
@@ -3,7 +3,6 @@ package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
@@ -11,15 +10,17 @@ import (
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// SSE event stream endpoints for the file:events and meta:events Redis Streams.
// SSE event stream endpoints for the file:events and meta:events streams.
//
// Wire contract (see docs/api-mediated-access.md):
// - One SSE event per Redis Stream entry.
// - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
// MUST treat it as a black-box string they echo back via Last-Event-ID.
// - One SSE event per stream entry.
// - `id:` is the opaque stream entry ID (`<ms>-<seq>`, the Redis Streams
// form, kept across the move to SQLite so cursors consumers persisted to
// disk still compare correctly); consumers MUST treat it as a black-box
// string they echo back via Last-Event-ID.
// - `event:` is the `type` field from the underlying entry.
// - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
// - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
@@ -30,30 +31,30 @@ import (
const (
sseHeartbeatInterval = 30 * time.Second
// sseReadBlock controls how long each XREAD blocks before returning.
// sseReadBlock controls how long each read waits for new entries.
// Much shorter than sseHeartbeatInterval so the handler stays responsive
// to client disconnects and the heartbeat check runs frequently enough
// that the keep-alive fires within a few seconds of its nominal cadence.
sseReadBlock = 2 * time.Second
// sseReadCount caps how many entries one XREAD returns. Small enough
// sseReadCount caps how many entries one read returns. Small enough
// to keep per-flush memory low, large enough to drain bursts cheaply.
sseReadCount = 100
)
// handleEventsFiles streams the file:events Redis Stream as SSE.
// handleEventsFiles streams the file:events stream as SSE.
func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "file:events")
}
// handleEventsMeta streams the meta:events Redis Stream as SSE.
// handleEventsMeta streams the meta:events stream as SSE.
func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "meta:events")
}
// streamSSE is the shared SSE pump shared by both endpoints. One handler per
// stream — there is no fan-out; each subscriber holds its own XREAD loop.
// That's fine at our scale (≤20 subscribers; one per consumer service).
// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads
// the stream itself and sleeps on the store's commit broadcast between reads —
// fine at our scale (≤20 subscribers; one per consumer service).
func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
if !s.storage.IsConnected() {
w.Header().Set("Retry-After", "5")
@@ -89,14 +90,14 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
w.WriteHeader(http.StatusOK)
flusher.Flush()
client := s.storage.GetRedisClient()
if client == nil {
store := s.storage.Store()
if store == nil {
// Storage was just disconnected — close cleanly.
return
}
ctx := r.Context()
cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher)
lastWrite := time.Now()
for {
@@ -104,37 +105,34 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
return
}
// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
// context's deadline isn't propagated into XREAD's BLOCK arg —
// that's a poll-style timeout on Redis itself — so we keep BLOCK
// short.
readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
result, err := client.XRead(readCtx, &redis.XReadArgs{
Streams: []string{stream, cursor},
Block: sseReadBlock,
Count: sseReadCount,
}).Result()
cancel()
// Take the wake-up channel BEFORE reading, so an append landing
// between the read and the wait still wakes us.
wake := store.Wait(stream)
entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount)
if err != nil {
// redis.Nil = BLOCK timeout with no entries. Anything else
// while the client is still connected is a real failure.
if errors.Is(err, redis.Nil) || isContextTimeout(err) {
// fall through to heartbeat check
} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
return
} else {
log.Printf("[SSE %s] XREAD error: %v", stream, err)
if ctx.Err() != nil {
return
}
log.Printf("[SSE %s] read error: %v", stream, err)
return
}
for _, str := range result {
for _, msg := range str.Messages {
writeSSEEntry(w, flusher, msg)
cursor = msg.ID
lastWrite = time.Now()
for _, e := range entries {
writeSSEEntry(w, flusher, e)
cursor = e.ID
lastWrite = time.Now()
}
if len(entries) == 0 {
timer := time.NewTimer(sseReadBlock)
select {
case <-ctx.Done():
timer.Stop()
return
case <-wake:
case <-timer.C:
}
timer.Stop()
}
// Heartbeat if quiet for >= the interval. Comment line; SSE
@@ -150,38 +148,44 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
}
// resolveStartCursor decides where to start reading.
// - No Last-Event-ID → start at "$" (only new entries).
// - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
// entries appended from now on). Redis' "$" was re-resolved on every
// XREAD, which dropped anything appended between two reads.
// - Last-Event-ID provided AND still in stream → resume from it.
// - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
// and resume from the oldest available entry's predecessor "0", so the
// next XREAD picks up everything still in the stream.
// next read picks up everything still in the stream.
//
// The "still in stream" check is cheap: one XRANGE for the first entry.
func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
if lastID == "" {
return "$"
}
// One-entry XRANGE to find the oldest ID currently in the stream.
// The "still in stream" check is cheap: one read of the first entry.
func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
if lastID == "" {
head, err := store.LastID(xCtx, stream)
if err != nil {
log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err)
return "0"
}
return head
}
oldestEntry, ok, err := store.Oldest(xCtx, stream)
if err != nil {
// Treat any error as "stream not available right now" and resume
// from the cursor — XREAD will either find it or block until new
// entries arrive.
log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
// from the cursor — the read either finds it or waits for new entries.
log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err)
return lastID
}
if len(entries) == 0 {
if !ok {
// Stream is empty. Cursor may be from a previous incarnation; safest
// is to resume from the cursor — XREAD against an empty stream
// returns nothing and blocks until new entries arrive.
// is to resume from the cursor — reading an empty stream returns
// nothing and waits until new entries arrive.
return lastID
}
oldest := entries[0].ID
oldest := oldestEntry.ID
if compareStreamIDs(lastID, oldest) >= 0 {
// Cursor is at or past the oldest — fine to resume.
return lastID
@@ -197,39 +201,35 @@ func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastI
fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
flusher.Flush()
// Return "0" so the next XREAD picks up entries starting from the
// Return "0" so the next read picks up entries starting from the
// stream's true minimum — including `oldest` itself.
return "0"
}
// writeSSEEntry serialises one Redis Stream entry as an SSE event.
// writeSSEEntry serialises one stream entry as an SSE event.
//
// `event:` comes from the entry's `type` field (no prefix munging). `data:`
// is a JSON object containing all OTHER fields. Field values from the Redis
// stream are interface{} (typically strings); we encode whatever the stream
// holds.
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
// is a JSON object containing all OTHER fields.
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) {
eventType := "message"
data := make(map[string]interface{}, len(msg.Values))
for k, v := range msg.Values {
data := make(map[string]string, len(e.Values))
for k, v := range e.Values {
if k == "type" {
if s, ok := v.(string); ok {
eventType = s
continue
}
eventType = v
continue
}
data[k] = v
}
payload, err := json.Marshal(data)
if err != nil {
log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err)
return
}
// Spec: each event ends with a blank line. id/event/data lines are
// terminated with single \n.
fmt.Fprintf(w, "id: %s\n", msg.ID)
fmt.Fprintf(w, "id: %s\n", e.ID)
fmt.Fprintf(w, "event: %s\n", eventType)
fmt.Fprintf(w, "data: %s\n\n", string(payload))
flusher.Flush()
@@ -263,18 +263,3 @@ func parseStreamID(id string) (int64, int64) {
}
return ms, seq
}
// isContextTimeout returns true for the context deadline error wrapped by
// go-redis when XREAD's read context fires. We treat it the same as
// redis.Nil — i.e. an empty poll cycle, not a fatal error.
func isContextTimeout(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
// go-redis sometimes wraps this in its own error type. String match is
// a safety net.
return strings.Contains(err.Error(), "context deadline exceeded")
}
internal/api/stats.go
+6 -4
View File
@@ -27,8 +27,10 @@ type fileStatsJSON struct {
// dashboard shows, in one request.
//
// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
// O(1) commands (SCARD, DBSIZE, INFO) plus a directory listing, recomputed on
// every call. Files and UDLUsers need a keyspace walk — ~0.5s for the file
// cheap (an index range count, a cached row count, a file stat) plus a
// directory listing, recomputed on every call. The `redis*` JSON names
// predate the move to SQLite and are kept for dashboard compatibility:
// redisKeys is the store's row count, redisMemory its on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
// from a background-refreshed cache and are null until the first sweep lands.
type StatsResponse struct {
@@ -47,7 +49,7 @@ type StatsResponse struct {
// statsCache holds the expensive half of /api/stats.
//
// Why a cache and not just "compute it": the dashboard polls. A 3s Redis walk
// Why a cache and not just "compute it": the dashboard polls. A multi-second store walk
// per tick, per open tab, is a self-inflicted outage on a box that is also
// serving playback — the same shape as meta-sort's /api/stats full sweep. So
// the handler NEVER waits on a walk: it returns whatever the last sweep
@@ -86,7 +88,7 @@ func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
c.running = false
// A failed sweep leaves the previous numbers in place rather than
// blanking the tiles; sweptAt still advances so a hard-failing walk
// (Redis down) does not spin once per poll.
// (store down) does not spin once per poll.
c.sweptAt = time.Now()
if files != nil {
c.files = files
internal/api/storage_helper_test.go
+19
View File
@@ -0,0 +1,19 @@
package api
import (
"path/filepath"
"testing"
"github.com/metazla/meta-core/internal/storage"
)
// newTestStorage opens a fresh metadata store in the test's temp dir.
func newTestStorage(t *testing.T) *storage.Client {
t.Helper()
c := storage.NewClient()
if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
t.Fatalf("open store: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return c
}
internal/api/udl.go
+3 -3
View File
@@ -7,7 +7,7 @@ package api
// meta-watch's local op-log. Records arrive already signed (and, for
// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
// version-gates and indexes them but never inspects the crypto. See
// internal/storage/udl.go for the Redis model.
// internal/storage/udl.go for the key model.
import (
"encoding/json"
@@ -65,7 +65,7 @@ type UDLPutBatchResponse struct {
Results []UDLPutBatchResult `json:"results"`
}
// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
// maxUDLBatch bounds one batch. Each record is a store write while the
// request is held open; sized to match the sign batch on the other side of the
// same operation (see maxSignBatch).
const maxUDLBatch = 1000
@@ -126,7 +126,7 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
Exists: exists,
}
// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
// Redis, a future writer bug) would otherwise fail mid-encode and emit a
// the store, a future writer bug) would otherwise fail mid-encode and emit a
// truncated body instead of a readable error.
if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
raw := json.RawMessage(cell.Value)
internal/api/udl_test.go
+1 -11
View File
@@ -6,9 +6,6 @@ import (
"net/http/httptest"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
// newUDLTestServer returns a Server wired to an in-memory Redis.
@@ -18,14 +15,7 @@ import (
// which the UDL handlers touch — they use s.storage and nothing else.
func newUDLTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return &Server{storage: c}
return &Server{storage: newTestStorage(t)}
}
// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
internal/config/config.go
+11 -7
View File
@@ -19,8 +19,8 @@ type Config struct {
BaseURL string // Base URL for stable service discovery
PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
// Redis configuration
RedisPort int // Redis port (default: 6379)
// Storage configuration
DBPathOverride string // META_CORE_DB_PATH; empty = <MetaCorePath>/db/meta-core.db
// HTTP API configuration
HTTPPort int // HTTP API port (default: 9000)
@@ -60,7 +60,7 @@ func Load() *Config {
APIPort: getEnvInt("API_PORT", 8180),
BaseURL: getEnv("BASE_URL", ""),
PublicURL: getEnv("META_CORE_PUBLIC_URL", ""),
RedisPort: getEnvInt("REDIS_PORT", 6379),
DBPathOverride: getEnv("META_CORE_DB_PATH", ""),
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
HTTPHost: getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
@@ -83,16 +83,20 @@ func Load() *Config {
return cfg
}
// LockFilePath returns the path to the Redis-volume mutex. Held by
// LockFilePath returns the path to the database-volume mutex (one meta-core
// per database file). Held by
// docker/leader-election.sh; nothing reads it, and it is NOT service
// discovery — see docs/project-architecture/beacon-v2.md.
func (c *Config) LockFilePath() string {
return c.MetaCorePath + "/locks/kv-leader.lock"
}
// RedisDataDir returns the path to Redis data directory
func (c *Config) RedisDataDir() string {
return c.MetaCorePath + "/db/redis"
// DBPath returns the path of meta-core's SQLite database file.
func (c *Config) DBPath() string {
if c.DBPathOverride != "" {
return c.DBPathOverride
}
return c.MetaCorePath + "/db/meta-core.db"
}
// MountsFilePath returns the path to the mounts configuration file
internal/events/meta_publisher.go
+61 -154
View File
@@ -1,199 +1,89 @@
package events
import (
"context"
"fmt"
"log"
"strings"
"sync"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/storage"
)
const (
// MetaEventsStream is the Redis stream name for metadata events
MetaEventsStream = "meta:events"
// MetaEventsStream is the stream name for metadata events.
MetaEventsStream = storage.MetaEventsStream
// MetaEventsMaxLen is the approximate cap on stream length. Bounded
// retention is a contract requirement for the SSE wire — see
// docs/api-mediated-access.md "Backing-store retention and lifecycle".
// Cheap (~) trim keeps growth bounded without serialising writes.
MetaEventsMaxLen = 100_000
MetaEventsMaxLen = storage.MetaEventsMaxLen
)
// MetaPublisher subscribes to Redis keyspace notifications
// and publishes metadata change events to the meta:events stream
// republishBatch is how many events one republish transaction appends —
// large enough to amortise the commit, small enough that the writer is never
// held for long.
const republishBatch = 5_000
// MetaPublisher owns the meta:events stream's bulk republish.
//
// Live change events no longer pass through here: under Redis this subscribed
// to keyspace notifications and copied each into the stream, a hop that could
// drop events whenever the subscriber lagged. The storage engine now appends
// the event in the same transaction as the change (internal/kv), so a change
// and its event commit together.
type MetaPublisher struct {
client *redis.Client
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
client *storage.Client
running bool
mu sync.RWMutex
}
// NewMetaPublisher creates a new MetaPublisher
func NewMetaPublisher(client *redis.Client) *MetaPublisher {
ctx, cancel := context.WithCancel(context.Background())
return &MetaPublisher{
client: client,
ctx: ctx,
cancel: cancel,
}
// NewMetaPublisher creates a new MetaPublisher.
func NewMetaPublisher(client *storage.Client) *MetaPublisher {
return &MetaPublisher{client: client}
}
// Start begins subscribing to keyspace notifications.
//
// The stream is NOT truncated on startup — the SSE wire contract requires
// Last-Event-ID resumption to survive a meta-core restart, which means
// entries must persist across restart. Growth is bounded instead by
// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
// Start marks the publisher ready. The stream is NOT truncated on startup —
// the SSE wire contract requires Last-Event-ID resumption to survive a
// meta-core restart. Growth is bounded by MetaEventsMaxLen instead.
func (p *MetaPublisher) Start() error {
p.mu.Lock()
if p.running {
p.mu.Unlock()
return nil
}
defer p.mu.Unlock()
p.running = true
p.mu.Unlock()
p.wg.Add(1)
go p.subscribeLoop()
log.Println("[MetaPublisher] Started - listening for keyspace notifications")
log.Println("[MetaPublisher] Started")
return nil
}
// Stop stops the publisher
// Stop stops the publisher.
func (p *MetaPublisher) Stop() error {
p.mu.Lock()
if !p.running {
p.mu.Unlock()
return nil
}
defer p.mu.Unlock()
p.running = false
p.mu.Unlock()
p.cancel()
p.wg.Wait()
log.Println("[MetaPublisher] Stopped")
return nil
}
// IsRunning returns true if the publisher is running
// IsRunning returns true if the publisher is running.
func (p *MetaPublisher) IsRunning() bool {
p.mu.RLock()
defer p.mu.RUnlock()
return p.running
}
// subscribeLoop subscribes to keyspace notifications and publishes events
func (p *MetaPublisher) subscribeLoop() {
defer p.wg.Done()
// Subscribe to all file:* keyspace events
// Pattern: __keyspace@0__:file:*
pattern := "__keyspace@0__:file:*"
pubsub := p.client.PSubscribe(p.ctx, pattern)
defer pubsub.Close()
log.Printf("[MetaPublisher] Subscribed to pattern: %s", pattern)
ch := pubsub.Channel()
for {
select {
case <-p.ctx.Done():
return
case msg := <-ch:
if msg == nil {
continue
}
// msg.Channel: __keyspace@0__:file:midhash256:abc123/tmdb
// msg.Payload: set, del, expire, etc.
p.publishEvent(msg.Channel, msg.Payload)
}
}
}
// publishEvent publishes a keyspace event to the meta:events stream
func (p *MetaPublisher) publishEvent(channel, operation string) {
// Extract key from channel: __keyspace@0__:file:xxx/field -> file:xxx/field
key := strings.TrimPrefix(channel, "__keyspace@0__:")
// Skip index key and non-file keys
if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
return
}
// Skip the per-record field index. It is bookkeeping (see
// storage.FieldsField), not a metadata field — publishing it would make
// every downstream consumer that parses `file:<id>/<field>` off this
// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
// field change on a key they can never read.
if strings.HasSuffix(key, "/"+storage.FieldsField) {
return
}
// Build event fields
fields := map[string]interface{}{
"type": operation, // "set", "del", "expire"
"key": key, // "file:midhash256:abc123/tmdb"
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
}
// Approximate MAXLEN trim keeps the stream bounded across restarts
// (see Start). Cap is intentionally generous; most consumers reconnect
// well within retention.
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
Stream: MetaEventsStream,
MaxLen: MetaEventsMaxLen,
Approx: true,
Values: fields,
}).Result()
if err != nil {
log.Printf("[MetaPublisher] Error publishing event: %v", err)
}
}
// publishEventDirect publishes an event directly (not from keyspace notification)
func (p *MetaPublisher) publishEventDirect(operation, key string) {
fields := map[string]interface{}{
"type": operation,
"key": key,
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
}
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
Stream: MetaEventsStream,
MaxLen: MetaEventsMaxLen,
Approx: true,
Values: fields,
}).Result()
if err != nil {
log.Printf("[MetaPublisher] Error publishing direct event: %v", err)
}
}
// RepublishFunc is a function type for getting metadata that can be republished
type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error)
// RepublishAllMetadata iterates over all existing file metadata and publishes events
// This is called on startup and can be triggered via API
// RepublishAllMetadata publishes a `set` event for every property of every
// record. Called on startup and via the API.
//
// It exists for consumers that rebuild their state by replaying meta:events
// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather
// than backfilling over HTTP. Appended in bounded batches, each its own
// transaction, so a multi-million-event republish never holds the writer.
func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) {
p.mu.RLock()
running := p.running
p.mu.RUnlock()
if !running {
if !p.IsRunning() {
return 0, fmt.Errorf("publisher not running")
}
// Get all hash IDs and metadata getter
hashIDs, getMetadata, err := republishFunc()
if err != nil {
return 0, fmt.Errorf("failed to get hash IDs: %w", err)
@@ -201,22 +91,39 @@ func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int,
log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs))
publishedCount := 0
published := 0
batch := make([]map[string]string, 0, republishBatch)
flush := func() error {
if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil {
return err
}
published += len(batch)
batch = batch[:0]
return nil
}
for _, hashID := range hashIDs {
// Get all properties for this file
metadata, err := getMetadata(hashID)
if err != nil || metadata == nil {
continue
}
// Publish a set event for each property
ts := fmt.Sprintf("%d", time.Now().UnixMilli())
for property := range metadata {
key := fmt.Sprintf("file:%s/%s", hashID, property)
p.publishEventDirect("set", key)
publishedCount++
batch = append(batch, map[string]string{
"type": "set",
"key": fmt.Sprintf("file:%s/%s", hashID, property),
"ts": ts,
})
if len(batch) >= republishBatch {
if err := flush(); err != nil {
return published, fmt.Errorf("republish: %w", err)
}
}
}
}
if err := flush(); err != nil {
return published, fmt.Errorf("republish: %w", err)
}
log.Printf("[MetaPublisher] Republished %d property events", publishedCount)
return publishedCount, nil
log.Printf("[MetaPublisher] Republished %d property events", published)
return published, nil
}
internal/identity/verify.go
+1 -1
View File
@@ -56,7 +56,7 @@ func ValidAction(a Action) bool {
}
// ValidUID exposes the uid shape check to callers outside this package (the
// API layer, which must reject a hostile uid before it reaches a Redis SCAN
// API layer, which must reject a hostile uid before it reaches a key-range read
// pattern or a file path).
func ValidUID(uid string) bool { return validUID(uid) }
internal/kv/kv.go
+713
View File
@@ -0,0 +1,713 @@
// Package kv is meta-core's storage engine: one string→string table in an
// embedded SQLite file.
//
// # Why a key/value table and not a schema
//
// meta-core's data model is schema-free on purpose — a record is a set of flat
// `file:<rid>/<field>` keys, and a new metadata field is just a new key
// (METADATA_KEYS.md). The engine keeps exactly that model:
//
// CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID
//
// is the only table that holds data, and it never needs a migration. What
// Redis gave us that a bare KV would not — "list a record's fields", "find
// records by field value", free-text search — comes from the table being
// *ordered* (a record is a key range) and from two derived structures the
// storage package maintains on top: an expression index over the field name
// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv
// rows and never authoritative.
//
// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a
// 169k-record corpus (Redis plus the API's in-memory copy needed to search
// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here
// and the working set is a bounded page cache.
//
// # Concurrency
//
// SQLite allows one writer at a time, so every write goes through one writer
// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED:
// the writer runs every request waiting at that moment inside one SQLite
// transaction, each under its own SAVEPOINT (a failing request rolls back
// alone), and commits once. A commit — writing the dirty pages of the table,
// the field index and the FTS shadow tables to the WAL — was ~40% of the cost
// of a record write, so sharing it is what makes concurrent ingest scale.
//
// Reads use a separate pool and see the last committed state (WAL). Inside
// Update, read through the *Tx — the reader pool cannot see uncommitted rows —
// and never call Update from inside an Update callback (the writer would wait
// for itself).
//
// # Events
//
// Redis keyspace notifications are replaced by the transaction itself: every
// Put/Delete of a key the Options.EventKey predicate selects is appended, in
// the same transaction, to Options.EventStream as {type: set|del, key, ts}.
// A change and its event therefore commit together or not at all.
package kv
import (
"context"
"database/sql"
"errors"
"fmt"
"log"
"net/url"
"os"
"strings"
"sync"
"time"
_ "modernc.org/sqlite"
)
// Options configures a Store.
type Options struct {
// EventStream receives a set/del entry for every changed key EventKey
// selects. Empty disables change events.
EventStream string
// EventMaxLen is the approximate retention of EventStream.
EventMaxLen int64
// EventKey selects the keys whose changes are published.
EventKey func(key string) bool
// WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection.
WriterCacheMB int
ReaderCacheMB int
// Readers is the size of the read connection pool.
Readers int
// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
// for derived indexes owned by the caller.
Schema string
}
// Store is an open database.
type Store struct {
path string
opts Options
w *sql.DB // single connection: the one writer
r *sql.DB // read pool
streamMu sync.Mutex
streams map[string]*streamState
// Hot write statements, prepared once on the writer connection: parsing
// SQL on every Put was the single largest cost of a record write.
stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
writes chan *writeReq
writeMu sync.Mutex // held by the writer while a group runs
closeOnce sync.Once
stopped chan struct{}
}
// writeReq is one Update call waiting for the writer.
type writeReq struct {
fn func(tx *Tx) error
done chan error
}
// maxGroup caps how many requests share one commit, bounding how long the
// writer holds a transaction open.
const maxGroup = 128
// Open opens (creating if needed) the database file at path.
func Open(path string, opts Options) (*Store, error) {
if opts.Readers <= 0 {
opts.Readers = 8
}
if opts.WriterCacheMB <= 0 {
opts.WriterCacheMB = 64
}
if opts.ReaderCacheMB <= 0 {
opts.ReaderCacheMB = 16
}
if opts.EventMaxLen <= 0 {
opts.EventMaxLen = 100_000
}
w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false))
if err != nil {
return nil, fmt.Errorf("open writer: %w", err)
}
w.SetMaxOpenConns(1)
w.SetMaxIdleConns(1)
w.SetConnMaxLifetime(0)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
w.Close()
return nil, fmt.Errorf("create kv: %w", err)
}
if opts.Schema != "" {
// No deadline: on an existing store, the first open after a new index
// is added builds it over every row (minutes at millions of keys).
// Cancelling that just rolls it back and restarts it on the next boot.
started := time.Now()
if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil {
w.Close()
return nil, fmt.Errorf("apply schema: %w", err)
}
if d := time.Since(started); d > 5*time.Second {
log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second))
}
}
r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true))
if err != nil {
w.Close()
return nil, fmt.Errorf("open readers: %w", err)
}
r.SetMaxOpenConns(opts.Readers)
r.SetMaxIdleConns(opts.Readers)
st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)}
// Fresh deadline: the schema step above may legitimately have taken minutes.
ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
for _, p := range []struct {
dst **sql.Stmt
query string
}{
{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
} {
if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil {
r.Close()
w.Close()
return nil, fmt.Errorf("prepare: %w", err)
}
}
st.writes = make(chan *writeReq)
st.stopped = make(chan struct{})
go st.writer()
return st, nil
}
// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class
// Redis' AOF-everysec gave us: a power loss may drop the last transactions,
// never corrupt the file.
func dsn(path string, cacheMB int, readOnly bool) string {
q := url.Values{}
q.Add("_pragma", "busy_timeout(30000)")
q.Add("_pragma", "journal_mode(WAL)")
q.Add("_pragma", "synchronous(NORMAL)")
q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024))
q.Add("_pragma", "temp_store(MEMORY)")
q.Add("_pragma", "journal_size_limit(268435456)")
if readOnly {
q.Add("_pragma", "query_only(1)")
} else {
q.Add("_txlock", "immediate")
}
return "file:" + path + "?" + q.Encode()
}
// Close stops the writer and closes both pools, checkpointing the WAL into the
// main file.
func (s *Store) Close() error {
s.closeOnce.Do(func() { close(s.stopped) })
s.writeMu.Lock() // wait for an in-flight group to finish
defer s.writeMu.Unlock()
_, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`)
return errors.Join(s.r.Close(), s.w.Close())
}
// Ping reports whether the database answers.
func (s *Store) Ping(ctx context.Context) error {
var one int
return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one)
}
// Path is the database file path.
func (s *Store) Path() string { return s.path }
// DB exposes the read pool for callers that own derived indexes (search). It
// is read-only (query_only), so it cannot bypass Update.
func (s *Store) DB() *sql.DB { return s.r }
// FileSize returns the on-disk size of the database plus its WAL.
func (s *Store) FileSize() int64 {
var total int64
for _, suffix := range []string{"", "-wal"} {
if fi, err := os.Stat(s.path + suffix); err == nil {
total += fi.Size()
}
}
return total
}
// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer
// slot and waits for readers, so callers run it when idle.
func (s *Store) Checkpoint(ctx context.Context) error {
_, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
return err
}
// KV is one key/value pair.
type KV struct {
K, V string
}
// Reader is the read surface shared by Store (committed state) and Tx (the
// transaction's own view). Storage helpers take a Reader so they work both
// inside and outside a write.
type Reader interface {
Get(ctx context.Context, k string) (string, bool, error)
MGet(ctx context.Context, keys []string) (map[string]string, error)
Range(ctx context.Context, prefix, after string, limit int) ([]KV, error)
Count(ctx context.Context, prefix string) (int64, error)
}
type queryer interface {
QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
}
// PrefixEnd returns the smallest string greater than every string with the
// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range.
// An empty result means "unbounded".
func PrefixEnd(prefix string) string {
b := []byte(prefix)
for i := len(b) - 1; i >= 0; i-- {
if b[i] < 0xff {
b[i]++
return string(b[:i+1])
}
}
return ""
}
func get(ctx context.Context, q queryer, k string) (string, bool, error) {
var v string
err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, err
}
return v, true, nil
}
// mgetBatch stays well under SQLite's default 32766 bound-parameter cap.
const mgetBatch = 500
func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) {
out := make(map[string]string, len(keys))
for start := 0; start < len(keys); start += mgetBatch {
end := min(start+mgetBatch, len(keys))
chunk := keys[start:end]
args := make([]any, len(chunk))
for i, k := range chunk {
args[i] = k
}
rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...)
if err != nil {
return nil, err
}
for rows.Next() {
var k, v string
if err := rows.Scan(&k, &v); err != nil {
rows.Close()
return nil, err
}
out[k] = v
}
if err := rows.Close(); err != nil {
return nil, err
}
}
return out, nil
}
// rangeQuery returns keys with prefix, strictly after `after` (when non-empty),
// in key order, at most limit (<= 0 means all).
func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) {
lo, loOp := prefix, ">="
if after != "" && after >= prefix {
lo, loOp = after, ">"
}
query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?`
args := []any{lo}
if end := PrefixEnd(prefix); end != "" {
query += ` AND k < ?`
args = append(args, end)
}
query += ` ORDER BY k`
if limit > 0 {
query += ` LIMIT ?`
args = append(args, limit)
}
rows, err := q.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []KV
for rows.Next() {
var kv KV
if err := rows.Scan(&kv.K, &kv.V); err != nil {
return nil, err
}
out = append(out, kv)
}
return out, rows.Err()
}
func count(ctx context.Context, q queryer, prefix string) (int64, error) {
query := `SELECT count(*) FROM kv WHERE k >= ?`
args := []any{prefix}
if end := PrefixEnd(prefix); end != "" {
query += ` AND k < ?`
args = append(args, end)
}
var n int64
err := q.QueryRowContext(ctx, query, args...).Scan(&n)
return n, err
}
// Get returns the committed value of k.
func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) }
// MGet returns the committed values of the keys that exist.
func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) {
return mget(ctx, s.r, keys)
}
// Range returns committed keys under prefix after `after`, in key order.
func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
return rangeQuery(ctx, s.r, prefix, after, limit)
}
// Count returns how many committed keys sit under prefix.
func (s *Store) Count(ctx context.Context, prefix string) (int64, error) {
return count(ctx, s.r, prefix)
}
// ScanPrefix walks every committed key under prefix in pages of `page`, each
// page its own short read so a long walk never pins a WAL snapshot (which
// would stop checkpoints and let the WAL grow without bound).
func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error {
if page <= 0 {
page = 5000
}
after := ""
for {
batch, err := rangeQuery(ctx, s.r, prefix, after, page)
if err != nil {
return err
}
if len(batch) == 0 {
return nil
}
if err := fn(batch); err != nil {
return err
}
if len(batch) < page {
return nil
}
after = batch[len(batch)-1].K
}
}
// CountAll counts every row. A full scan — callers cache it.
func (s *Store) CountAll(ctx context.Context) (int64, error) {
var n int64
err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n)
return n, err
}
// ErrClosed is returned by Update on a closed store.
var ErrClosed = errors.New("kv: store closed")
// Update runs fn atomically: its writes (and the change events they publish)
// commit together or not at all. fn may share a SQLite transaction with other
// concurrent Updates (group commit) but is isolated from their failures by a
// savepoint. Never call Update from inside fn.
//
// ctx bounds only the wait for the writer. Once fn starts it runs to
// completion: interrupting a statement inside a shared transaction would roll
// back every request in the group.
func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error {
req := &writeReq{fn: fn, done: make(chan error, 1)}
select {
case s.writes <- req:
case <-ctx.Done():
return ctx.Err()
case <-s.stopped:
return ErrClosed
}
return <-req.done
}
// writer is the single goroutine that executes writes.
func (s *Store) writer() {
for {
var first *writeReq
select {
case first = <-s.writes:
case <-s.stopped:
return
}
group := []*writeReq{first}
drain:
for len(group) < maxGroup {
select {
case r := <-s.writes:
group = append(group, r)
default:
break drain
}
}
s.writeMu.Lock()
s.runGroup(group)
s.writeMu.Unlock()
}
}
// group is the state shared by the requests of one commit.
type group struct {
tx *sql.Tx
pending map[string]*streamPending
stmts map[*sql.Stmt]*sql.Stmt
}
func (g *group) snapshot() map[string]streamPending {
out := make(map[string]streamPending, len(g.pending))
for k, p := range g.pending {
out[k] = *p
}
return out
}
func (g *group) restore(snap map[string]streamPending) {
g.pending = make(map[string]*streamPending, len(snap))
for k, p := range snap {
p := p
g.pending[k] = &p
}
}
func (s *Store) runGroup(reqs []*writeReq) {
ctx := context.Background()
results := make([]error, len(reqs))
fail := func(err error) {
for i, r := range reqs {
if results[i] == nil {
results[i] = err
}
r.done <- results[i]
}
}
sqlTx, err := s.w.BeginTx(ctx, nil)
if err != nil {
fail(fmt.Errorf("begin: %w", err))
return
}
g := &group{tx: sqlTx}
for i, r := range reqs {
if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil {
_ = sqlTx.Rollback()
fail(fmt.Errorf("savepoint: %w", err))
return
}
snap := g.snapshot()
tx := &Tx{ctx: ctx, s: s, g: g}
err := r.fn(tx)
if err == nil {
err = tx.flushEvents()
}
if err != nil {
results[i] = err
if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
_ = sqlTx.Rollback()
fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err))
return
}
g.restore(snap)
}
if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil {
_ = sqlTx.Rollback()
fail(fmt.Errorf("release savepoint: %w", err))
return
}
}
if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
_ = sqlTx.Rollback()
fail(err)
return
}
if err := sqlTx.Commit(); err != nil {
fail(fmt.Errorf("commit: %w", err))
return
}
s.afterCommit(g.pending)
for i, r := range reqs {
r.done <- results[i]
}
}
// Tx is one Update's view of the write transaction. Its reads see its own
// writes (and those of earlier requests in the same group).
type Tx struct {
ctx context.Context
s *Store
g *group
events []map[string]string
}
func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
// stmt binds one of the store's prepared statements to the transaction.
func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt {
if t.g.stmts == nil {
t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5)
}
bound := t.g.stmts[st]
if bound == nil {
bound = t.g.tx.StmtContext(t.ctx, st)
t.g.stmts[st] = bound
}
return bound
}
// Context is the context the transaction runs under.
func (t *Tx) Context() context.Context { return t.ctx }
func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) {
var v string
err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v)
if errors.Is(err, sql.ErrNoRows) {
return "", false, nil
}
if err != nil {
return "", false, err
}
return v, true, nil
}
func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) {
return mget(ctx, t.sqlTx(), keys)
}
func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
return rangeQuery(ctx, t.sqlTx(), prefix, after, limit)
}
func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) {
return count(ctx, t.sqlTx(), prefix)
}
// Exec runs a raw statement in the transaction — for the caller's derived
// tables (FTS), never for kv rows (which would skip events).
func (t *Tx) Exec(query string, args ...any) (sql.Result, error) {
return t.sqlTx().ExecContext(t.ctx, query, args...)
}
// QueryRow runs a raw single-row query in the transaction.
func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
return t.sqlTx().QueryRowContext(t.ctx, query, args...)
}
// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
// keyspace notifications did.
func (t *Tx) Put(k, v string) error {
if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
return fmt.Errorf("put %q: %w", k, err)
}
t.event("set", k)
return nil
}
// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
// Publishes nothing when the key already existed (nothing changed).
func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
if err != nil {
return false, fmt.Errorf("put %q: %w", k, err)
}
n, _ := res.RowsAffected()
if n > 0 {
t.event("set", k)
}
return n > 0, nil
}
// Delete removes k. Reports whether it existed; publishes `del` only then,
// as Redis did.
func (t *Tx) Delete(k string) (bool, error) {
res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
if err != nil {
return false, fmt.Errorf("delete %q: %w", k, err)
}
n, _ := res.RowsAffected()
if n > 0 {
t.event("del", k)
}
return n > 0, nil
}
// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
// returns how many went. Bulk callers loop over separate Updates with a limit
// so one transaction never grows to millions of rows.
func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
end := PrefixEnd(prefix)
sel := `SELECT k FROM kv WHERE k >= ?`
args := []any{prefix}
if end != "" {
sel += ` AND k < ?`
args = append(args, end)
}
if limit > 0 {
sel += ` ORDER BY k LIMIT ?`
args = append(args, limit)
}
rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
if err != nil {
return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
}
var keys []string
for rows.Next() {
var k string
if err := rows.Scan(&k); err != nil {
rows.Close()
return 0, err
}
keys = append(keys, k)
}
if err := rows.Close(); err != nil {
return 0, err
}
for _, k := range keys {
t.event("del", k)
}
return int64(len(keys)), nil
}
func (t *Tx) event(op, k string) {
if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) {
return
}
t.events = append(t.events, map[string]string{
"type": op,
"key": k,
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
})
}
// flushEvents appends the queued change events to the event stream, inside
// the transaction, right before commit.
func (t *Tx) flushEvents() error {
if len(t.events) == 0 {
return nil
}
evs := t.events
t.events = nil
for _, e := range evs {
if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil {
return err
}
}
return nil
}
internal/kv/kv_test.go
+314
View File
@@ -0,0 +1,314 @@
package kv
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
"time"
)
func openTest(t *testing.T, path string) *Store {
t.Helper()
if path == "" {
path = filepath.Join(t.TempDir(), "kv.db")
}
s, err := Open(path, Options{
EventStream: "meta:events",
EventMaxLen: 100_000,
EventKey: func(k string) bool { return strings.HasPrefix(k, "file:") },
})
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = s.Close() })
return s
}
func update(t *testing.T, s *Store, fn func(tx *Tx) error) {
t.Helper()
if err := s.Update(context.Background(), fn); err != nil {
t.Fatalf("update: %v", err)
}
}
func events(t *testing.T, s *Store) []Entry {
t.Helper()
es, err := s.ReadAfter(context.Background(), "meta:events", "0", 0)
if err != nil {
t.Fatalf("read events: %v", err)
}
return es
}
// The replacement for keyspace notifications: a change and its event commit
// together, with Redis' semantics — `set` even for an unchanged value, `del`
// only for a key that existed, and nothing for keys outside the event scope.
func TestChangesPublishEventsWithRedisSemantics(t *testing.T) {
s := openTest(t, "")
update(t, s, func(tx *Tx) error {
_ = tx.Put("file:A/title", "x")
_ = tx.Put("file:A/title", "x") // unchanged value still publishes
_ = tx.Put("cid:bafk", "A") // outside the event scope
_, _ = tx.Delete("file:A/missing")
_, _ = tx.Delete("file:A/title")
return nil
})
got := events(t, s)
want := []string{"set file:A/title", "set file:A/title", "del file:A/title"}
if len(got) != len(want) {
t.Fatalf("events = %+v, want %v", got, want)
}
for i, e := range got {
if e.Values["type"]+" "+e.Values["key"] != want[i] {
t.Errorf("event %d = %v, want %q", i, e.Values, want[i])
}
}
}
func TestRolledBackChangesPublishNothing(t *testing.T) {
s := openTest(t, "")
err := s.Update(context.Background(), func(tx *Tx) error {
_ = tx.Put("file:A/title", "x")
return errors.New("boom")
})
if err == nil {
t.Fatal("want the error back")
}
if _, ok, _ := s.Get(context.Background(), "file:A/title"); ok {
t.Fatal("rolled-back write is visible")
}
if es := events(t, s); len(es) != 0 {
t.Fatalf("rolled-back write published %+v", es)
}
}
func TestDeletePrefixPublishesOneDelPerKey(t *testing.T) {
s := openTest(t, "")
update(t, s, func(tx *Tx) error {
for _, k := range []string{"file:A/a", "file:A/b", "file:AB/c"} {
_ = tx.Put(k, "1")
}
return nil
})
var n int64
update(t, s, func(tx *Tx) error {
var err error
n, err = tx.DeletePrefix("file:A/", 0)
return err
})
if n != 2 {
t.Fatalf("deleted %d, want 2 (file:AB/ is not under file:A/)", n)
}
es := events(t, s)
if dels := es[3:]; len(dels) != 2 || dels[0].Values["type"] != "del" || dels[1].Values["type"] != "del" {
t.Fatalf("want two del events, got %+v", es)
}
}
// IDs are Redis-shaped and strictly increasing, within one millisecond and
// across a reopen — the SSE cursors consumers persist depend on both.
func TestStreamIDsAreMonotonicAcrossReopen(t *testing.T) {
path := filepath.Join(t.TempDir(), "kv.db")
s, err := Open(path, Options{})
if err != nil {
t.Fatal(err)
}
var ids []string
if err := s.Update(context.Background(), func(tx *Tx) error {
for i := 0; i < 50; i++ {
id, err := tx.Append("file:events", 0, map[string]string{"n": "x"})
if err != nil {
return err
}
ids = append(ids, id)
}
return nil
}); err != nil {
t.Fatal(err)
}
_ = s.Close()
s2 := openTest(t, path)
var after string
update(t, s2, func(tx *Tx) error {
var err error
after, err = tx.Append("file:events", 0, map[string]string{"n": "y"})
return err
})
ids = append(ids, after)
for i := 1; i < len(ids); i++ {
if !idLess(t, ids[i-1], ids[i]) {
t.Fatalf("id %d (%s) does not follow %s", i, ids[i], ids[i-1])
}
}
if !strings.Contains(ids[0], "-") || strings.HasPrefix(ids[0], "0") {
t.Fatalf("id %q is not the unpadded <ms>-<seq> form", ids[0])
}
}
func idLess(t *testing.T, a, b string) bool {
t.Helper()
am, as, err := ParseID(a)
if err != nil {
t.Fatal(err)
}
bm, bs, err := ParseID(b)
if err != nil {
t.Fatal(err)
}
return am < bm || (am == bm && as < bs)
}
// The watcher's reset clears file:events. The next ID must still exceed every
// ID issued before the clear, or a consumer holding an old cursor would skip
// the reset event.
func TestClearStreamKeepsTheHighWaterMark(t *testing.T) {
s := openTest(t, "")
var before, after string
update(t, s, func(tx *Tx) error {
var err error
for i := 0; i < 5; i++ {
if before, err = tx.Append("file:events", 0, map[string]string{"type": "add"}); err != nil {
return err
}
}
return nil
})
update(t, s, func(tx *Tx) error { return tx.ClearStream("file:events") })
if n, _ := s.Len(context.Background(), "file:events"); n != 0 {
t.Fatalf("len after clear = %d", n)
}
update(t, s, func(tx *Tx) error {
var err error
after, err = tx.Append("file:events", 0, map[string]string{"type": "reset"})
return err
})
if !idLess(t, before, after) {
t.Fatalf("post-clear id %s does not follow pre-clear %s", after, before)
}
got, err := s.ReadAfter(context.Background(), "file:events", before, 10)
if err != nil || len(got) != 1 || got[0].Values["type"] != "reset" {
t.Fatalf("an old cursor must see the reset: %+v err=%v", got, err)
}
}
// MAXLEN ~ semantics across many small transactions (the common case: one
// event per write).
func TestStreamTrimsAcrossTransactions(t *testing.T) {
defer func(old int64) { trimEvery = old }(trimEvery)
trimEvery = 100
s := openTest(t, "")
const maxLen = 50
for i := int64(0); i < 3*trimEvery; i++ {
update(t, s, func(tx *Tx) error {
_, err := tx.Append("x", maxLen, map[string]string{"i": "1"})
return err
})
}
n, err := s.Len(context.Background(), "x")
if err != nil {
t.Fatal(err)
}
if n > maxLen+trimEvery {
t.Fatalf("stream holds %d entries, want <= %d", n, maxLen+trimEvery)
}
oldest, ok, err := s.Oldest(context.Background(), "x")
if err != nil || !ok {
t.Fatal(err)
}
if last, _ := s.LastID(context.Background(), "x"); !idLess(t, oldest.ID, last) {
t.Fatalf("oldest %s should precede last %s", oldest.ID, last)
}
}
// A subscriber that took the wake channel before reading is woken by the next
// commit — the property the SSE pump relies on to not miss an append.
func TestWaitIsClosedByTheNextCommit(t *testing.T) {
s := openTest(t, "")
wake := s.Wait("meta:events")
update(t, s, func(tx *Tx) error { return tx.Put("file:A/x", "1") })
select {
case <-wake:
case <-time.After(2 * time.Second):
t.Fatal("commit did not wake the waiter")
}
}
func TestReadAfterZeroFormsReadFromTheStart(t *testing.T) {
s := openTest(t, "")
update(t, s, func(tx *Tx) error {
_, err := tx.Append("x", 0, map[string]string{"n": "1"})
return err
})
for _, id := range []string{"", "0", "0-0"} {
got, err := s.ReadAfter(context.Background(), "x", id, 10)
if err != nil || len(got) != 1 {
t.Errorf("ReadAfter(%q) = %d entries, err %v; want 1", id, len(got), err)
}
}
}
func TestRangeIsExactlyThePrefix(t *testing.T) {
s := openTest(t, "")
update(t, s, func(tx *Tx) error {
for _, k := range []string{"file:A/a", "file:A/b/c", "file:A0", "file:AB/x", "file:A"} {
_ = tx.Put(k, "v")
}
return nil
})
rows, err := s.Range(context.Background(), "file:A/", "", 0)
if err != nil {
t.Fatal(err)
}
if len(rows) != 2 || rows[0].K != "file:A/a" || rows[1].K != "file:A/b/c" {
t.Fatalf("range = %+v", rows)
}
}
// Concurrent Updates share commits; a failing one must roll back alone and
// leave its neighbours' writes and events intact.
func TestGroupCommitIsolatesFailures(t *testing.T) {
s := openTest(t, "")
const n = 200
errs := make(chan error, n)
for i := 0; i < n; i++ {
go func(i int) {
errs <- s.Update(context.Background(), func(tx *Tx) error {
k := fmt.Sprintf("file:R%03d/title", i)
if err := tx.Put(k, "x"); err != nil {
return err
}
if i%10 == 0 {
return errors.New("fail")
}
return nil
})
}(i)
}
failed := 0
for i := 0; i < n; i++ {
if <-errs != nil {
failed++
}
}
if failed != n/10 {
t.Fatalf("failed = %d, want %d", failed, n/10)
}
rows, _ := s.Range(context.Background(), "file:", "", 0)
if len(rows) != n-n/10 {
t.Fatalf("committed %d rows, want %d", len(rows), n-n/10)
}
if es := events(t, s); len(es) != n-n/10 {
t.Fatalf("events = %d, want %d (one per committed write, none for failures)", len(es), n-n/10)
}
for _, r := range rows {
var i int
fmt.Sscanf(r.K, "file:R%03d/title", &i)
if i%10 == 0 {
t.Fatalf("failed request's write %s committed", r.K)
}
}
}
internal/kv/stream.go
+356
View File
@@ -0,0 +1,356 @@
package kv
// Append-only event streams, as rows of the kv table.
//
// A stream entry is the row
//
// ev:<stream>/<ms:13>-<seq:10> → JSON object of the entry's string fields
//
// Zero padding makes key order equal ID order, so reading "after an ID" is a
// plain range read and trimming is a range delete. On the wire IDs are the
// unpadded Redis form (`1700000000000-0`): meta-sort, meta-share and
// meta-stremio persist SSE cursors to disk across the Redis → SQLite switch,
// and those compare as <ms>-<seq> pairs.
//
// The last issued ID is kept in `evhwm:<stream>` and updated in the same
// transaction as the append. That high-water mark, not the newest surviving
// row, seeds the next ID — so Clear (the watcher's `reset`) never reissues an
// ID an old cursor would treat as already seen.
import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
)
// Entry is one stream entry.
type Entry struct {
ID string
Values map[string]string
}
type streamID struct{ ms, seq int64 }
func (id streamID) String() string {
return strconv.FormatInt(id.ms, 10) + "-" + strconv.FormatInt(id.seq, 10)
}
func (id streamID) padded() string { return fmt.Sprintf("%013d-%010d", id.ms, id.seq) }
func (id streamID) less(o streamID) bool {
return id.ms < o.ms || (id.ms == o.ms && id.seq < o.seq)
}
// ParseID parses a Redis-style stream ID. "", "0", "0-0", "-" all parse as the
// zero ID (before everything). A bare "<ms>" means "<ms>-0".
func ParseID(s string) (ms, seq int64, err error) {
if s == "" || s == "-" || s == "0" {
return 0, 0, nil
}
msPart, seqPart, hasSeq := strings.Cut(s, "-")
ms, err = strconv.ParseInt(msPart, 10, 64)
if err != nil || ms < 0 {
return 0, 0, fmt.Errorf("invalid stream id %q", s)
}
if hasSeq {
seq, err = strconv.ParseInt(seqPart, 10, 64)
if err != nil || seq < 0 {
return 0, 0, fmt.Errorf("invalid stream id %q", s)
}
}
return ms, seq, nil
}
func streamPrefix(stream string) string { return "ev:" + stream + "/" }
func hwmKey(stream string) string { return "evhwm:" + stream }
// streamState is the in-memory view of a stream's high-water mark plus the
// broadcast channel subscribers wait on. Updated only after a commit.
type streamState struct {
loaded bool
hwm streamID
sinceTrim int64 // committed appends since the last trim
notify chan struct{}
}
// streamPending is what a transaction did to one stream, applied to
// streamState on commit (and discarded on rollback).
type streamPending struct {
hwm streamID
appends int64
trimmed bool
lastTrimAt int64 // value of appends when this transaction last trimmed
}
func (s *Store) state(stream string) *streamState {
st := s.streams[stream]
if st == nil {
st = &streamState{notify: make(chan struct{})}
s.streams[stream] = st
}
return st
}
// Wait returns a channel closed by the next commit that appends to stream.
// Take it BEFORE reading, so an append between the read and the wait is not
// missed.
func (s *Store) Wait(stream string) <-chan struct{} {
s.streamMu.Lock()
defer s.streamMu.Unlock()
return s.state(stream).notify
}
func (s *Store) afterCommit(pending map[string]*streamPending) {
if len(pending) == 0 {
return
}
s.streamMu.Lock()
defer s.streamMu.Unlock()
for name, p := range pending {
st := s.state(name)
st.loaded = true
st.hwm = p.hwm
if p.trimmed {
st.sinceTrim = p.appends - p.lastTrimAt
} else {
st.sinceTrim += p.appends
}
close(st.notify)
st.notify = make(chan struct{})
}
}
// hwm returns the stream's last issued ID as seen by this transaction.
func (t *Tx) hwm(stream string) (streamID, error) {
if p := t.g.pending[stream]; p != nil {
return p.hwm, nil
}
t.s.streamMu.Lock()
st := t.s.state(stream)
loaded, hwm := st.loaded, st.hwm
t.s.streamMu.Unlock()
if loaded {
return hwm, nil
}
v, ok, err := t.Get(t.ctx, hwmKey(stream))
if err != nil {
return streamID{}, err
}
if !ok {
return streamID{}, nil
}
ms, seq, err := ParseID(v)
if err != nil {
return streamID{}, err
}
return streamID{ms, seq}, nil
}
// Append adds an entry to stream and returns its ID. The stream is trimmed to
// about maxLen entries (maxLen <= 0: unbounded).
func (t *Tx) Append(stream string, maxLen int64, fields map[string]string) (string, error) {
last, err := t.hwm(stream)
if err != nil {
return "", fmt.Errorf("stream %s hwm: %w", stream, err)
}
id := streamID{ms: time.Now().UnixMilli()}
if !last.less(id) {
id = streamID{ms: last.ms, seq: last.seq + 1}
}
body, err := json.Marshal(fields)
if err != nil {
return "", err
}
if _, err := t.stmt(t.s.stInsert).ExecContext(t.ctx, streamPrefix(stream)+id.padded(), string(body)); err != nil {
return "", fmt.Errorf("append %s: %w", stream, err)
}
if t.g.pending == nil {
t.g.pending = make(map[string]*streamPending)
}
p := t.g.pending[stream]
if p == nil {
p = &streamPending{}
t.g.pending[stream] = p
}
p.hwm = id
p.appends++
// Trim in steps, not per append: MAXLEN ~ semantics, like the XADD it
// replaces. The count spans transactions (most append one event each).
if maxLen > 0 {
t.s.streamMu.Lock()
since := t.s.state(stream).sinceTrim
t.s.streamMu.Unlock()
if !p.trimmed {
since += p.appends
} else {
since = p.appends - p.lastTrimAt
}
if since >= trimEvery {
if err := t.trim(stream, maxLen); err != nil {
return "", err
}
p.trimmed, p.lastTrimAt = true, p.appends
}
}
return id.String(), nil
}
// trimEvery is how many appends pass between trims. A trim locates its cut
// point by walking maxLen entries back from the newest, so it is kept rare;
// the cost is overshooting maxLen by at most trimEvery entries.
var trimEvery int64 = 10_000 // var so tests can lower it
func (t *Tx) trim(stream string, maxLen int64) error {
prefix := streamPrefix(stream)
var cut string
err := t.sqlTx().QueryRowContext(t.ctx,
`SELECT k FROM kv WHERE k >= ? AND k < ? ORDER BY k DESC LIMIT 1 OFFSET ?`,
prefix, PrefixEnd(prefix), maxLen).Scan(&cut)
if err != nil {
return nil // fewer than maxLen entries (sql.ErrNoRows): nothing to trim
}
_, err = t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k <= ?`, prefix, cut)
return err
}
// flushHWM persists each touched stream's high-water mark — once per
// transaction, not per append.
func (t *Tx) flushHWM() error {
for stream, p := range t.g.pending {
if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, hwmKey(stream), p.hwm.String()); err != nil {
return fmt.Errorf("stream %s hwm: %w", stream, err)
}
}
return nil
}
// TrimAll trims stream to maxLen inside the transaction regardless of how many
// appends this transaction made. Used after an append burst split over many
// short transactions.
func (t *Tx) TrimAll(stream string, maxLen int64) error { return t.trim(stream, maxLen) }
// ClearStream deletes every entry of stream, keeping its high-water mark.
func (t *Tx) ClearStream(stream string) error {
prefix := streamPrefix(stream)
_, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k < ?`, prefix, PrefixEnd(prefix))
return err
}
func decodeEntry(stream, k, v string) (Entry, error) {
padded := strings.TrimPrefix(k, streamPrefix(stream))
ms, seq, err := ParseID(padded)
if err != nil {
return Entry{}, err
}
var values map[string]string
if err := json.Unmarshal([]byte(v), &values); err != nil {
return Entry{}, fmt.Errorf("decode %s: %w", k, err)
}
return Entry{ID: streamID{ms, seq}.String(), Values: values}, nil
}
func decodeEntries(stream string, rows []KV) ([]Entry, error) {
out := make([]Entry, 0, len(rows))
for _, kv := range rows {
e, err := decodeEntry(stream, kv.K, kv.V)
if err != nil {
return nil, err
}
out = append(out, e)
}
return out, nil
}
// ReadAfter returns up to n entries strictly after id, oldest first. "0",
// "0-0" and "" read from the beginning.
func (s *Store) ReadAfter(ctx context.Context, stream, id string, n int) ([]Entry, error) {
ms, seq, err := ParseID(id)
if err != nil {
return nil, err
}
prefix := streamPrefix(stream)
after := ""
if ms != 0 || seq != 0 {
after = prefix + streamID{ms, seq}.padded()
}
rows, err := rangeQuery(ctx, s.r, prefix, after, n)
if err != nil {
return nil, err
}
return decodeEntries(stream, rows)
}
// Oldest returns the oldest surviving entry, or ok=false for an empty stream.
func (s *Store) Oldest(ctx context.Context, stream string) (Entry, bool, error) {
rows, err := rangeQuery(ctx, s.r, streamPrefix(stream), "", 1)
if err != nil || len(rows) == 0 {
return Entry{}, false, err
}
e, err := decodeEntry(stream, rows[0].K, rows[0].V)
return e, err == nil, err
}
// LastID returns the last ID ever issued on stream ("0-0" if none) — the
// cursor meaning "only entries appended from now on".
func (s *Store) LastID(ctx context.Context, stream string) (string, error) {
s.streamMu.Lock()
st := s.state(stream)
loaded, hwm := st.loaded, st.hwm
s.streamMu.Unlock()
if loaded {
return hwm.String(), nil
}
v, ok, err := s.Get(ctx, hwmKey(stream))
if err != nil {
return "", err
}
if !ok {
return "0-0", nil
}
return v, nil
}
// Len is the number of surviving entries.
func (s *Store) Len(ctx context.Context, stream string) (int64, error) {
return count(ctx, s.r, streamPrefix(stream))
}
// RevRange returns up to n entries newest first, stopping at (excluding)
// entries at or before afterID when afterID is non-empty.
func (s *Store) RevRange(ctx context.Context, stream, afterID string, n int) ([]Entry, error) {
prefix := streamPrefix(stream)
query := `SELECT k, v FROM kv WHERE k > ? AND k < ? ORDER BY k DESC`
lo := prefix
if afterID != "" {
ms, seq, err := ParseID(afterID)
if err != nil {
return nil, err
}
lo = prefix + streamID{ms, seq}.padded()
}
args := []any{lo, PrefixEnd(prefix)}
if n > 0 {
query += ` LIMIT ?`
args = append(args, n)
}
rows, err := s.r.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var kvs []KV
for rows.Next() {
var kv KV
if err := rows.Scan(&kv.K, &kv.V); err != nil {
return nil, err
}
kvs = append(kvs, kv)
}
if err := rows.Err(); err != nil {
return nil, err
}
return decodeEntries(stream, kvs)
}
internal/leader/election.go
+4 -4
View File
@@ -13,7 +13,7 @@ import (
type LeaderLockInfo struct {
Hostname string `json:"hostname"`
BaseUrl string `json:"baseUrl"`
ApiUrl string `json:"apiUrl"` // meta-core API (port 9000)
ApiUrl string `json:"apiUrl"` // meta-core API (port 9000)
RedisUrl string `json:"redisUrl"`
WebdavUrl string `json:"webdavUrl"` // External WebDAV URL (via baseUrl)
WebdavUrlInternal string `json:"webdavUrlInternal"` // Internal WebDAV URL (direct to port 9000)
@@ -72,9 +72,9 @@ func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo {
webdavUrlInternal := fmt.Sprintf("http://%s:%d/webdav", hostname, p.config.HTTPPort)
return &LeaderLockInfo{
Hostname: hostname,
BaseUrl: baseUrl,
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
Hostname: hostname,
BaseUrl: baseUrl,
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
// RedisUrl deliberately empty — api-mediated-access PR D removes
// direct Redis exposure. Consumers route metadata I/O through
// meta-core's HTTP API (/meta/{hash}*) and event streams through
internal/schema/classify.go
+1 -1
View File
@@ -21,7 +21,7 @@ import (
// (crc32: "bafk3eaqerjhiwhy").
var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
// ClassifyValue returns the (primitive, hint) for a Redis value.
// ClassifyValue returns the (primitive, hint) for a stored value.
// exists=false (key missing) and JSON null literal both map to PrimUndefined.
// Empty string is a valid string.
func ClassifyValue(value string, exists bool) (Primitive, Hint) {
internal/schema/indexer.go
+89 -188
View File
@@ -6,17 +6,20 @@ import (
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/metazla/meta-core/internal/events"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
// fieldState is the per-field accumulator. perKey lets us undo a previous
// contribution when the same Redis key is re-SET or DELed.
// rebuildInterval is how often the schema is recomputed in the background.
// It describes the *shape* of the data (which fields exist, their types), which
// drifts slowly; a few minutes of staleness is invisible to the editor that
// reads it, and POST /api/schema/rescan recomputes on demand.
const rebuildInterval = 10 * time.Minute
// fieldState is the per-field accumulator.
type fieldState struct {
counts map[string]int
primitives map[Primitive]int
@@ -24,12 +27,6 @@ type fieldState struct {
typedCount int
undefinedCount int
languageKeyed bool
perKey map[string]labelPair
}
type labelPair struct {
prim Primitive
hint Hint
}
func newFieldState() *fieldState {
@@ -37,19 +34,25 @@ func newFieldState() *fieldState {
counts: make(map[string]int),
primitives: make(map[Primitive]int),
hintCounts: make(map[Hint]int),
perKey: make(map[string]labelPair),
}
}
// Indexer maintains a live field-schema by consuming meta:events.
// Indexer maintains the per-field schema by scanning the store.
//
// It used to follow meta:events and keep, per field, a map of EVERY record key
// it had seen (so a re-set or delete could undo that key's contribution). At
// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident
// memory once the search copy was gone — and it only worked if every event was
// replayed into it, which is half of why meta-core republished every field on
// boot. A periodic streaming scan holds one accumulator per FIELD instead and
// is exact by construction.
type Indexer struct {
client *redis.Client
keyPrefix string
store *kv.Store
mu sync.RWMutex
fields map[string]*fieldState
mu sync.RWMutex
snap *SchemaResponse
processed uint64 // total events processed since start
rebuildMu sync.Mutex
ctx context.Context
cancel context.CancelFunc
@@ -58,37 +61,41 @@ type Indexer struct {
running bool
}
// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
// so the indexer can build absolute Redis keys for GET calls.
func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
// NewIndexer creates an Indexer over store.
func NewIndexer(store *kv.Store) *Indexer {
ctx, cancel := context.WithCancel(context.Background())
return &Indexer{
client: client,
keyPrefix: keyPrefix,
fields: make(map[string]*fieldState),
ctx: ctx,
cancel: cancel,
}
return &Indexer{store: store, ctx: ctx, cancel: cancel}
}
// Start begins consuming the meta:events stream from the earliest available ID.
// Start computes the schema in the background now and every rebuildInterval.
func (i *Indexer) Start() error {
i.runMu.Lock()
defer i.runMu.Unlock()
if i.running {
i.runMu.Unlock()
return nil
}
i.running = true
i.runMu.Unlock()
i.wg.Add(1)
go i.consumeLoop()
log.Println("[SchemaIndexer] Started - consuming meta:events stream")
go func() {
defer i.wg.Done()
ticker := time.NewTicker(rebuildInterval)
defer ticker.Stop()
for {
if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil {
log.Printf("[SchemaIndexer] rebuild failed: %v", err)
}
select {
case <-i.ctx.Done():
return
case <-ticker.C:
}
}
}()
log.Println("[SchemaIndexer] Started - scanning the store")
return nil
}
// Stop terminates the consumer goroutine.
// Stop terminates the background rebuilds.
func (i *Indexer) Stop() error {
i.runMu.Lock()
if !i.running {
@@ -104,93 +111,55 @@ func (i *Indexer) Stop() error {
return nil
}
// ProcessedCount returns the total number of stream events processed.
// Used by WaitForDrain to detect when a rescan flood has been absorbed.
func (i *Indexer) ProcessedCount() uint64 {
return atomic.LoadUint64(&i.processed)
}
// Rebuild recomputes the schema from every record key and publishes it. One
// rebuild runs at a time; a concurrent caller waits and gets the fresh result.
func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) {
i.rebuildMu.Lock()
defer i.rebuildMu.Unlock()
// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
// the timeout elapses. Suitable for use right after RepublishMetadata.
func (i *Indexer) WaitForDrain(timeout time.Duration) {
deadline := time.Now().Add(timeout)
last := i.ProcessedCount()
stableSince := time.Now()
for time.Now().Before(deadline) {
time.Sleep(100 * time.Millisecond)
now := i.ProcessedCount()
if now != last {
last = now
stableSince = time.Now()
continue
}
if time.Since(stableSince) >= 300*time.Millisecond {
return
}
}
}
func (i *Indexer) consumeLoop() {
defer i.wg.Done()
lastID := "0"
for {
select {
case <-i.ctx.Done():
return
default:
}
res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
Streams: []string{events.MetaEventsStream, lastID},
Block: 2 * time.Second,
Count: 500,
}).Result()
if err != nil {
if err == redis.Nil || i.ctx.Err() != nil {
started := time.Now()
fields := make(map[string]*fieldState)
err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
for _, row := range rows {
fieldPath, keyHint, ok := parseFieldPath(row.K)
if !ok {
continue
}
log.Printf("[SchemaIndexer] XRead error: %v", err)
time.Sleep(1 * time.Second)
continue
}
for _, stream := range res {
for _, msg := range stream.Messages {
lastID = msg.ID
i.processEvent(msg.Values)
atomic.AddUint64(&i.processed, 1)
fs := fields[fieldPath]
if fs == nil {
fs = newFieldState()
fields[fieldPath] = fs
}
if keyHint != "" {
fs.languageKeyed = true
}
prim, hint := ClassifyValue(row.V, true)
incContribution(fs, labelPair{prim: prim, hint: hint})
}
return ctx.Err()
})
if err != nil {
return nil, err
}
snap := &SchemaResponse{
Fields: make(map[string]*FieldSchema, len(fields)),
GeneratedAt: time.Now().UTC(),
Source: "scan",
}
for fp, fs := range fields {
snap.Fields[fp] = computeFieldSchema(fs)
}
i.mu.Lock()
i.snap = snap
i.mu.Unlock()
log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond))
return snap, nil
}
func (i *Indexer) processEvent(values map[string]any) {
op, _ := values["type"].(string)
key, _ := values["key"].(string)
if key == "" {
return
}
fieldPath, keyHint, ok := parseFieldPath(key)
if !ok {
return
}
switch op {
case "set":
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
cancel()
exists := err != redis.Nil
if err != nil && err != redis.Nil {
// Treat transient errors as "value disappeared" — nothing to record.
return
}
prim, hint := ClassifyValue(val, exists)
i.upsert(fieldPath, key, keyHint, prim, hint)
case "del", "expire":
i.remove(fieldPath, key)
}
type labelPair struct {
prim Primitive
hint Hint
}
// parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
@@ -222,46 +191,6 @@ func parseFieldPath(key string) (string, KeyHint, bool) {
return property, "", true
}
func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
i.mu.Lock()
defer i.mu.Unlock()
fs, ok := i.fields[fieldPath]
if !ok {
fs = newFieldState()
i.fields[fieldPath] = fs
}
if keyHint != "" {
fs.languageKeyed = true
}
if old, ok := fs.perKey[redisKey]; ok {
decContribution(fs, old)
}
fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
incContribution(fs, labelPair{prim: prim, hint: hint})
}
func (i *Indexer) remove(fieldPath, redisKey string) {
i.mu.Lock()
defer i.mu.Unlock()
fs, ok := i.fields[fieldPath]
if !ok {
return
}
old, ok := fs.perKey[redisKey]
if !ok {
return
}
delete(fs.perKey, redisKey)
decContribution(fs, old)
if len(fs.perKey) == 0 {
delete(i.fields, fieldPath)
}
}
func incContribution(fs *fieldState, lp labelPair) {
label := breakdownLabel(lp.prim, lp.hint)
fs.counts[label]++
@@ -276,43 +205,15 @@ func incContribution(fs *fieldState, lp labelPair) {
}
}
func decContribution(fs *fieldState, lp labelPair) {
label := breakdownLabel(lp.prim, lp.hint)
fs.counts[label]--
if fs.counts[label] <= 0 {
delete(fs.counts, label)
}
if lp.prim == PrimUndefined {
fs.undefinedCount--
return
}
fs.typedCount--
fs.primitives[lp.prim]--
if fs.primitives[lp.prim] <= 0 {
delete(fs.primitives, lp.prim)
}
if lp.hint != "" {
fs.hintCounts[lp.hint]--
if fs.hintCounts[lp.hint] <= 0 {
delete(fs.hintCounts, lp.hint)
}
}
}
// Snapshot returns the current schema. Safe to call concurrently with consumption.
// Snapshot returns the latest computed schema. Before the first scan finishes
// it is empty with Source "pending".
func (i *Indexer) Snapshot() *SchemaResponse {
i.mu.RLock()
defer i.mu.RUnlock()
out := &SchemaResponse{
Fields: make(map[string]*FieldSchema, len(i.fields)),
GeneratedAt: time.Now().UTC(),
Source: "live",
if i.snap == nil {
return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"}
}
for fp, fs := range i.fields {
out.Fields[fp] = computeFieldSchema(fs)
}
return out
return i.snap
}
func computeFieldSchema(fs *fieldState) *FieldSchema {
internal/schema/indexer_test.go
+56
View File
@@ -0,0 +1,56 @@
package schema
import (
"context"
"path/filepath"
"testing"
"github.com/metazla/meta-core/internal/kv"
)
func TestRebuildDerivesTypesFromTheStore(t *testing.T) {
store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{})
if err != nil {
t.Fatal(err)
}
defer store.Close()
if err := store.Update(context.Background(), func(tx *kv.Tx) error {
for k, v := range map[string]string{
"file:A/sizeByte": "100",
"file:B/sizeByte": "200",
"file:A/title/eng": "One",
"file:B/title/fre": "Deux",
"file:A/mixed": "1",
"file:B/mixed": "x",
"cid:bafknotrecord": "A",
} {
if err := tx.Put(k, v); err != nil {
return err
}
}
return nil
}); err != nil {
t.Fatal(err)
}
ix := NewIndexer(store)
if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 {
t.Fatalf("before a scan: %+v", got)
}
snap, err := ix.Rebuild(context.Background())
if err != nil {
t.Fatal(err)
}
if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 {
t.Fatalf("sizeByte = %+v", f)
}
if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode {
t.Fatalf("title = %+v (want language-keyed)", f)
}
if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed {
t.Fatalf("mixed = %+v", f)
}
if len(snap.Fields) != 3 {
t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields)
}
}
internal/snapshot/export.go
+1 -1
View File
@@ -24,7 +24,7 @@ type ExportOptions struct {
SourceHost string
}
// Exporter writes snapshot ZIPs from a live Redis store.
// Exporter writes snapshot ZIPs from the live metadata store.
type Exporter struct {
storage *storage.Client
}
internal/snapshot/flatten.go
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"strings"
)
// Flat keys use "/" as the separator. Property paths inside Redis are
// Flat keys use "/" as the separator. Property paths inside the store are
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
const flatSep = "/"
@@ -33,7 +33,7 @@ func Reconstruct(flat map[string]string) interface{} {
}
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
// flat property map whose values are all strings (Redis-ready).
// flat property map whose values are all strings (store-ready).
//
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
// values are stored as "" to preserve the field's presence — this matches
@@ -127,7 +127,7 @@ func flattenInto(v interface{}, path string, out map[string]string) {
out[path] = ""
case map[string]interface{}:
if len(t) == 0 && path != "" {
// Empty objects don't have a Redis representation; skip.
// Empty objects don't have a flat-key representation; skip.
return
}
for k, child := range t {
internal/snapshot/import.go
+2 -2
View File
@@ -73,7 +73,7 @@ type ImportResult struct {
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
}
// Importer applies snapshot ZIPs to a live Redis store.
// Importer applies snapshot ZIPs to the live metadata store.
type Importer struct {
storage *storage.Client
}
@@ -83,7 +83,7 @@ func NewImporter(stor *storage.Client) *Importer {
}
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
// *bytes.Reader after buffering the upload) and applies it to Redis.
// *bytes.Reader after buffering the upload) and applies it to the store.
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
if i.storage == nil || !i.storage.IsConnected() {
return nil, fmt.Errorf("storage not connected")
internal/snapshot/wipe.go
+1 -1
View File
@@ -16,7 +16,7 @@ type WipeResult struct {
MetadataDeleted int64 `json:"metadataDeleted"`
}
// Wiper clears metadata from Redis.
// Wiper clears metadata from the store.
type Wiper struct {
storage *storage.Client
}
internal/storage/cid_resolution.go
+172 -284
View File
@@ -5,10 +5,9 @@ import (
"fmt"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/cid"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// This file implements the CID-resolution layer that lives on top of the
@@ -18,7 +17,7 @@ import (
// The UUID is an internal implementation detail; the mental model is
// "N CIDs → one record of keys→metadata".
// - Every known CID for a file is registered as a reverse-index entry:
// cid:<bareCid> → <uuid> (Redis STRING)
// cid:<bareCid> → <uuid>
// and as a member of the per-root bare-CID key-set:
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
@@ -27,14 +26,13 @@ import (
// *derived by rank* from the key-set on read (GetMetadataDocument), never
// stored.
//
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
// any write of a cids/<cid> field and register the reverse-index entry, so
// The record write path (writeFieldsTx) catches any write of a cids/<cid>
// field and registers the reverse-index entry in the same transaction, so
// writers that just emit the key-set members don't need to know about the
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
// already held by the caller, avoiding nested RLock (which can deadlock
// against a waiting writer).
// reverse index at all. The helpers here take the caller's *kv.Tx: a write
// must never open a second transaction (there is one writer connection).
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
// CIDIndexPrefix is the key prefix for the reverse index. Combined
// with a bare CID, it forms the full key: "cid:bafk…".
const CIDIndexPrefix = "cid:"
@@ -45,152 +43,112 @@ const CIDIndexPrefix = "cid:"
// reverse-index entries that need cleanup).
const CIDsKeyPrefix = "cids/"
// DuplicatesField is the per-root SET key holding additional file paths
// whose content hashes to the same CIDs as this root. The watcher writes
// here when it sees a new path for content that's already indexed; meta-dup
// reads from it.
const DuplicatesField = "duplicates"
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
// cidStr is a bare multibase CIDv1 ("bafk…").
func (c *Client) buildCIDIndexKey(cidStr string) string {
return c.buildKey(CIDIndexPrefix + cidStr)
}
// Duplicate paths — other physical locations whose content hashes to the same
// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
// writes one when it sees a new path for content that's already indexed.
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
// in the file index. Returns the new UUID. The caller is expected to follow
// in the root index. Returns the new UUID. The caller is expected to follow
// up with AddAlias for at least one CID — a root without any alias is
// reachable only via the index, which works but defeats the point.
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
var uuid string
err := c.update(func(tx *kv.Tx) error {
var err error
uuid, err = mintTx(tx, filePath, size, mtimeNano)
return err
})
return uuid, err
}
func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
uuid, err := NewUUID()
if err != nil {
return "", fmt.Errorf("uuid: %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
pipe := c.client.Pipeline()
pipe.Set(ctx, prefix+"filePath", filePath, 0)
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
if _, err := pipe.Exec(ctx); err != nil {
return "", fmt.Errorf("mint pipeline: %w", err)
}
return uuid, nil
return uuid, writeFieldsTx(tx, uuid, map[string]string{
"filePath": filePath,
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
})
}
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
// Idempotent: registering the same CID twice is a no-op.
func (c *Client) AddAlias(uuid, cidStr string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.addAliasLocked(ctx, uuid, cidStr)
}
// addAliasLocked is the lock-free body of AddAlias. The caller must already
// hold c.mu (read or write). Split out so the alias-maintenance hooks
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
// the work without re-entering the lock — nested RLock can deadlock against
// a waiting writer.
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
return c.update(func(tx *kv.Tx) error {
return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
})
}
keysetField := CIDsKeyPrefix + cidStr
keysetKey := c.buildKeyPrefix(uuid) + keysetField
// Every branch below writes the key-set member, so index the field name
// once here — including the two guard branches that skip the alias itself.
// Missing it would leave `cids/<cid>` invisible to a record read.
defer c.addFieldLocked(ctx, uuid, keysetField)
// addAliasTx registers the reverse-index entry for a cids/<cid> member the
// caller is writing (writeFieldsTx writes the member itself).
func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
// Self-pointing alias guard. If the caller is trying to register
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
// dual-root failure mode in disguise — it means somebody wrote
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
// write; leave any existing alias alone. Still record the key-set
// membership so the field-level write succeeds.
// write; leave any existing alias alone. The key-set membership is still
// recorded by the caller, so the field-level write succeeds.
if uuid == cidStr {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
return nil
}
// Guard against the dual-root pattern: if cid:<cid> already points at a
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
// just record the key-set membership so it's still discoverable via the
// root → cids enumeration.
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != "" && existing != uuid {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
}
pipe := c.client.Pipeline()
pipe.Set(ctx, indexKey, uuid, 0)
pipe.Set(ctx, keysetKey, "true", 0)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("add alias pipeline: %w", err)
}
return nil
}
// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller must already hold c.mu, and must delete
// the cids/<cid> key-set member itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat
// file:<uuid>/cids/<cid>, the __fields__ index entry, and this reverse alias —
// while the delete path only ever removed the first two. A "deleted" CID
// therefore still resolved: GET /meta/<cid>, ResolveRoot and GetByCID all kept
// answering with the record. The asymmetry is not theoretical. meta-share
// published dag-pb roots computed over RAR volumes and par2 files onto six
// records on watch.nsl.sh; removing the member was expressible, removing the
// claim was not, so the false identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasLocked, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err == redis.Nil {
return nil
}
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
// key-set membership keeps the CID discoverable via root → cids.
indexKey := CIDIndexPrefix + cidStr
existing, _, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != uuid {
// Somebody else owns this alias. Leave it.
if existing != "" && existing != uuid {
return nil
}
return c.client.Del(ctx, indexKey).Err()
if existing == uuid {
return nil
}
return tx.Put(indexKey, uuid)
}
// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
// itself (DeleteProperty does both).
//
// ⚠ Why this exists. A cids/ member write touches two keys — the flat
// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
// only removed the first. A "deleted" CID therefore still resolved: GET
// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
// The asymmetry is not theoretical. meta-share published dag-pb roots computed
// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
// the member was expressible, removing the claim was not, so the false
// identity survived its own retraction.
//
// Guarded, not blind: the alias is removed only when it currently points at
// THIS uuid. Two records can legitimately race to claim a cid (see the
// dual-root guard in addAliasTx, which leaves the first writer's alias in
// place); deleting a member from the loser must not unpick the winner's index.
func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return nil
}
indexKey := CIDIndexPrefix + cidStr
existing, ok, err := tx.Get(tx.Context(), indexKey)
if err != nil {
return fmt.Errorf("read existing alias: %w", err)
}
if !ok || existing != uuid {
// Absent, or somebody else owns this alias. Leave it.
return nil
}
_, err = tx.Delete(indexKey)
return err
}
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
@@ -210,8 +168,8 @@ func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) err
// - hash matches a reverse-index entry → return UUID
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
//
// Errors are converted to "no resolution" — if Redis is down or the lookup
// fails we let the caller proceed with the bare hash and surface the Redis
// Errors are converted to "no resolution" — if the store is down or the
// lookup fails we let the caller proceed with the bare hash and surface the
// failure on the actual operation.
func (c *Client) ResolveRoot(hash string) string {
if hash == "" {
@@ -228,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string {
// when the CID is unknown; the empty string is the "not found" signal rather
// than a typed error so callers can branch without errors.As.
func (c *Client) GetByCID(cidStr string) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return "", err
}
if cidStr == "" {
return "", nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
ctx, cancel := opCtx()
defer cancel()
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
if err == redis.Nil {
return "", nil
}
uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
if err != nil {
return "", fmt.Errorf("get cid index: %w", err)
}
@@ -252,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
}
// AddDuplicatePath records that path is another physical location whose
// content matches uuid. Returns true if path was newly added (not already
// in the set). The watcher calls this when it sees a file whose midhash
// matches an existing root but at a different filePath.
// content matches uuid. Returns true if path was newly added. The watcher
// calls this when it sees a file whose midhash matches an existing root but
// at a different filePath.
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
if uuid == "" || path == "" {
return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
if err != nil {
return false, fmt.Errorf("sadd duplicates: %w", err)
}
return added > 0, nil
added := false
err := c.update(func(tx *kv.Tx) error {
var err error
added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
return err
})
return added, err
}
// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
// prefix stripped). Caller must hold c.mu.
//
// Reads the field index and filters by prefix; only an un-indexed root falls
// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
// stripped).
func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
prefix := recordPrefix(uuid) + CIDsKeyPrefix
rows, err := r.Range(ctx, prefix, "", 0)
if err != nil {
return nil, err
}
out := make([]string, 0, len(rows))
for _, row := range rows {
out = append(out, row.K[len(prefix):])
}
return out, nil
}
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
// (including the cids/<cid> key-set), every reverse-index entry for those
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
// is a successful no-op.
// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
// with no remaining keys is a successful no-op.
func (c *Client) DeleteRoot(uuid string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
if uuid == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
// the list of reverse-index entries to clean up.
cids, err := c.cidsForRootLocked(ctx, uuid)
if err != nil {
return c.update(func(tx *kv.Tx) error {
_, err := deleteRecordTx(tx, uuid)
return err
}
// 2. Delete every reverse-index entry in one shot.
if len(cids) > 0 {
keys := make([]string, 0, len(cids))
for _, cidStr := range cids {
keys = append(keys, c.buildCIDIndexKey(cidStr))
}
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del cid index: %w", err)
}
}
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return fmt.Errorf("scan root keys: %w", err)
}
if len(keys) > 0 {
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del root keys: %w", err)
}
}
cursor = next
if cursor == 0 {
break
}
}
// 4. Remove from the file index set.
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
return fmt.Errorf("srem index: %w", err)
}
// Marked so the incremental search-index refresh reads it, finds it gone,
// and evicts it — otherwise a deleted root keeps answering searches until
// the next full rebuild.
c.markDirtyLocked(ctx, uuid)
return nil
}
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
// root. If it's a key-set member — field name starts with "cids/" — it
// registers the implied CID in the reverse index. Writers emit cids/<cid>
// members without knowing about the reverse index; this hook keeps it in
// sync transparently.
//
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
// (logged, not returned): an out-of-sync reverse index is recoverable via a
// sweep, while a failed property write is not.
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
cidStr := cidFromKeysetField(field)
if cidStr == "" {
return nil
}
return c.addAliasLocked(ctx, uuid, cidStr)
}
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
// over every entry in a metadata map. Used by SetMetadataFlat /
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
// hold c.mu.
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
for k := range metadata {
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
}
})
}
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
@@ -406,24 +275,23 @@ type MetadataDocument struct {
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
// never existed).
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
if uuid == "" {
return nil, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := opCtx()
defer cancel()
flat, err := c.getMetadataFlatInternal(ctx, uuid)
flat, err := readRecord(ctx, s, uuid)
if err != nil {
return nil, err
}
prefix := c.buildKeyPrefix(uuid)
if flat == nil {
flat = make(map[string]string)
}
// Hoist the cids/<cid> key-set members into a typed array and derive the
// canonical CID by rank. The members remain in Flat too — that's how
@@ -443,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
}
}
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers duplicates: %w", err)
dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
if err != nil {
return nil, fmt.Errorf("read duplicates: %w", err)
}
dups := make([]string, 0, len(dupRows))
for _, row := range dupRows {
dups = append(dups, row.K[len(dupPrefix(uuid)):])
}
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
@@ -474,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas
if midhash == "" {
return "", false, fmt.Errorf("midhash is required")
}
uuid, err := c.GetByCID(midhash)
if err != nil {
return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if uuid == "" {
uuid, err = c.Mint(absPath, size, mtimeNano)
var (
uuid string
minted bool
)
// One transaction: the lookup and the mint cannot interleave with another
// registration of the same content, so two concurrent scans of one file
// can no longer mint two roots for it.
err := c.update(func(tx *kv.Tx) error {
ctx := tx.Context()
existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
if err != nil {
return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
return fmt.Errorf("GetByCID(%s): %w", midhash, err)
}
if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
uuid = existing
if uuid == "" {
if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
return fmt.Errorf("Mint(%s): %w", absPath, err)
}
minted = true
if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
}
return nil
}
return uuid, true, nil
}
existing, _ := c.GetProperty(uuid, "filePath")
if existing != absPath {
if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
if err != nil {
return err
}
return uuid, false, nil
if current != absPath {
if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
}
return nil
}
if err := writeFieldsTx(tx, uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return nil
})
if err != nil {
return "", false, err
}
if _, err := c.MergeMetadataFlat(uuid, map[string]string{
"sizeByte": strconv.FormatInt(size, 10),
"mtimeNano": strconv.FormatInt(mtimeNano, 10),
}); err != nil {
return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
}
return uuid, false, nil
return uuid, minted, nil
}
internal/storage/cid_retraction_test.go
+18 -17
View File
@@ -6,13 +6,13 @@ import (
)
// Tests for retracting a `cids/<cid>` key-set member (cid_resolution.go's
// removeAliasLocked, reached through DeleteProperty).
// removeAliasTx, reached through DeleteProperty).
//
// Why this is worth its own file. Adding a member writes THREE keys — the flat
// `file:<uuid>/cids/<cid>`, the `__fields__` index entry, and the reverse alias
// `cid:<cid>` → `<uuid>`. Until removeAliasLocked existed, deleting removed only
// the first two, so a retracted CID kept resolving to the record: the member was
// gone from the document and the claim was not.
// Why this is worth its own file. Adding a member writes two keys — the flat
// `file:<uuid>/cids/<cid>` and the reverse alias `cid:<cid>` → `<uuid>`. Until
// removeAliasLocked (now removeAliasTx) existed, deleting removed only the
// first, so a retracted CID kept resolving to the record: the member was gone
// from the document and the claim was not.
//
// That asymmetry had a live consequence. meta-share minted dag-pb roots and
// midhash256s over whatever `archive::pick_media` elected, which on an
@@ -26,19 +26,20 @@ import (
// aliasTarget returns the uuid that cid:<cidStr> currently resolves to, or "".
func aliasTarget(t *testing.T, c *Client, cidStr string) string {
t.Helper()
got, err := c.client.Get(context.Background(), c.buildCIDIndexKey(cidStr)).Result()
got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr)
if err != nil {
return ""
t.Fatalf("get alias: %v", err)
}
return got
}
func hasField(t *testing.T, c *Client, hashID, field string) bool {
t.Helper()
ok, err := c.client.SIsMember(context.Background(), c.buildFieldsKey(hashID), field).Result()
fields, err := c.GetMetadataFlat(hashID)
if err != nil {
t.Fatalf("sismember: %v", err)
t.Fatalf("read record: %v", err)
}
_, ok := fields[field]
return ok
}
@@ -50,7 +51,7 @@ const (
)
func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const uuid = "01KSBHMTESTRECORD"
if err := c.SetMetadataFlat(uuid, map[string]string{
@@ -77,13 +78,13 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
}
// 1. the flat key
if v, err := c.client.Get(context.Background(),
c.buildKeyPrefix(uuid)+"cids/"+poisonedRoot).Result(); err == nil {
if v, ok, _ := c.Store().Get(context.Background(),
recordPrefix(uuid)+"cids/"+poisonedRoot); ok {
t.Fatalf("flat key should be gone, still %q", v)
}
// 2. the field index
// 2. the record read
if hasField(t, c, uuid, "cids/"+poisonedRoot) {
t.Fatal("field index still lists the retracted member")
t.Fatal("record read still lists the retracted member")
}
// 3. the reverse alias — the one that used to survive
if got := aliasTarget(t, c, poisonedRoot); got != "" {
@@ -110,7 +111,7 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
// writer's alias in place. Retracting the loser's member must not unpick the
// winner's index — which is why removeAliasLocked compares before deleting.
func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const winner = "01KSBHMWINNER"
const loser = "01KSBHMLOSER"
const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m"
@@ -140,7 +141,7 @@ func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
// A non-cids field has no alias to clean up, and the delete must not invent one.
func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
withClient(t, func(t *testing.T, c *Client) {
const uuid = "01KSBHMORDINARY"
if err := c.SetMetadataFlat(uuid, map[string]string{
"cids/" + goodMidhash: "true",
internal/storage/clear_test.go
+105 -91
View File
@@ -1,111 +1,122 @@
package storage
import (
"context"
"path/filepath"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/kv"
)
// newTestClient spins an in-memory Redis and returns a connected Client with
// the given namespace prefix. The server is torn down with the test.
func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
// newTestClient opens a fresh database in the test's temp dir.
func newTestClient(t *testing.T) *Client {
t.Helper()
s := miniredis.RunT(t)
c := NewClient(prefix)
if err := c.Connect("redis://" + s.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
c := NewClient()
if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return c, s
return c
}
// allKeys lists every row (test helper; full scan).
func allKeys(t *testing.T, c *Client) []string {
t.Helper()
var keys []string
if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
for _, r := range rows {
keys = append(keys, r.K)
}
return nil
}); err != nil {
t.Fatalf("scan: %v", err)
}
return keys
}
// TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
// leak: ClearAllMetadata used to delete every file:<root>/* key and the
// file:__index__ set, but left the cid:<cid> → root reverse-index entries
// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
// a surviving entry silently redirects a later write into a deleted root.
// leak: ClearAllMetadata used to delete every record key and the root index,
// but left the cid:<cid> → root reverse-index entries behind. Those stale
// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry
// silently redirects a later write into a deleted root.
func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
for _, prefix := range []string{"", "mm:"} {
name := "no-prefix"
if prefix != "" {
name = "prefixed"
c := newTestClient(t)
// Two records, each with several CID aliases (a midhash plus siblings) —
// exactly what the watcher produces.
aliases := map[string][]string{}
for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
if err != nil {
t.Fatalf("mint: %v", err)
}
t.Run(name, func(t *testing.T) {
c, s := newTestClient(t, prefix)
cids := []string{"bafkmid" + uuid, "bafkfull" + uuid}
for _, cidStr := range cids {
if err := c.AddAlias(uuid, cidStr); err != nil {
t.Fatalf("add alias %s: %v", cidStr, err)
}
}
if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil {
t.Fatalf("add duplicate: %v", err)
}
aliases[uuid] = cids
}
// Two records, each with several CID aliases (a midhash plus
// siblings) — exactly what the watcher produces.
aliases := map[string][]string{}
for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
if err != nil {
t.Fatalf("mint: %v", err)
}
cids := []string{
"bafkmid" + uuid,
"bafkfull" + uuid,
}
for _, cidStr := range cids {
if err := c.AddAlias(uuid, cidStr); err != nil {
t.Fatalf("add alias %s: %v", cidStr, err)
}
}
aliases[uuid] = cids
}
// An entry already orphaned by a previous (buggy) clear: a reverse-index
// key whose root no longer exists. The sweep is expected to collect it too.
if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil {
t.Fatalf("seed orphan: %v", err)
}
// An entry already orphaned by a previous (buggy) clear: a
// reverse-index key whose root no longer exists. The sweep is
// expected to collect it too.
if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
t.Fatalf("seed orphan: %v", err)
for uuid, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != uuid {
t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
}
}
}
// Sanity: the aliases resolve before the clear.
for uuid, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != uuid {
t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid)
}
}
}
deleted, err := c.ClearAllMetadata()
if err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
if deleted != 2 {
t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
}
deleted, err := c.ClearAllMetadata()
if err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
// deletedCount counts files, not keys.
if deleted != 2 {
t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
// Nothing record-shaped may survive: not the records, not their aliases,
// duplicates or search rows, not the pre-existing orphan. (The event
// streams and the schema sentinel are not metadata and stay.)
for _, k := range allKeys(t, c) {
for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} {
if strings.HasPrefix(k, ns) {
t.Errorf("key survived the clear: %q", k)
}
}
}
var ftsRows int
if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 {
t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err)
}
// No reverse-index key may survive — not for the cleared records,
// not the pre-existing orphan.
if leaked := s.Keys(); len(leaked) != 0 {
t.Errorf("keys survived the clear: %v", leaked)
for _, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != cidStr {
t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
}
// And the aliases must no longer resolve: ResolveRoot returns the
// hash unchanged when there is no index entry.
for _, cids := range aliases {
for _, cidStr := range cids {
if got := c.ResolveRoot(cidStr); got != cidStr {
t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got)
}
}
}
if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
}
})
}
}
if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
}
}
// TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
// so a clear on a prefixed client cannot reach another namespace's index.
// keys outside the metadata namespaces (magnet cache, coverage markers, a key
// that merely CONTAINS "cid:") must survive a metadata wipe.
func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
c, s := newTestClient(t, "mm:")
c := newTestClient(t)
uuid, err := c.Mint("watch/a.mkv", 100, 1000)
if err != nil {
@@ -114,25 +125,28 @@ func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
if err := c.AddAlias(uuid, "bafkmine"); err != nil {
t.Fatalf("add alias: %v", err)
}
// Another namespace's reverse-index entry, plus an unrelated key.
if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
t.Fatalf("seed foreign cid: %v", err)
}
if err := s.Set("unrelated", "keep-me"); err != nil {
t.Fatalf("seed unrelated: %v", err)
for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} {
if err := c.SetRawValue(k, v); err != nil {
t.Fatalf("seed %s: %v", k, err)
}
}
if _, err := c.ClearAllMetadata(); err != nil {
t.Fatalf("ClearAllMetadata: %v", err)
}
for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
if !s.Exists(k) {
t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} {
if _, _, ok, _ := c.GetKeyInfo(k); !ok {
t.Errorf("clear deleted foreign key %q", k)
}
}
if s.Exists(c.buildCIDIndexKey("bafkmine")) {
if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok {
t.Errorf("own reverse-index entry survived the clear")
}
}
// withClient runs fn against a fresh client.
func withClient(t *testing.T, fn func(t *testing.T, c *Client)) {
t.Helper()
fn(t, newTestClient(t))
}
internal/storage/client.go
+673 -1234
View File
File diff suppressed because it is too large Load Diff
internal/storage/domain_screen_migration.go
-193
View File
@@ -1,193 +0,0 @@
package storage
import (
"context"
"fmt"
"time"
)
// The one-shot corpus sweep for METADATA_KEYS.md §14.17: the `film` + `tv`
// domains merged into `screen`, and the standalone-vs-serial split they were
// really carrying moved to the new `workForm` key.
//
// # Why a sweep and not a read-side alias
//
// `domain` is **stored, not derived on read** — that is what lets the
// `contentKind` vocabulary stay open (a peer that has never heard of a kind
// still routes the record correctly). The price of storing it is that a
// vocabulary change has to be paid once, in the data: every consumer's filter
// is an exact match, so a record left saying `film` is invisible to a
// `domain:screen` wall, and teaching every consumer to accept both spellings
// forever is exactly the alternation the merge exists to delete.
//
// So the values are rewritten at rest, here, and no alias vocabulary survives
// in query filters. (meta-search additionally normalises on the way into its
// index, because a record can still arrive from a peer that has not been swept
// — that is a courtesy for un-swept peers, not a second source of truth.)
//
// # Contract
//
// Idempotent: a record already carrying `screen` and a `workForm` is skipped,
// so the endpoint is safe to call repeatedly and safe to call on a fresh box.
// Returns the number of records **changed**, not the number examined.
// legacyDomainReplacements maps a retired `domain` value to its replacement.
var legacyDomainReplacements = map[string]string{
"film": "screen",
"tv": "screen",
}
// workFormByContentKind is the `contentKind` → `workForm` table from
// METADATA_KEYS.md §1.
//
// ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain` (and of
// meta-search's copy). It exists here only to backfill the pre-merge corpus —
// meta-core never classifies a record it is given, and nothing else in Go reads
// it. If the tables ever disagree, the writers win: this one is a one-shot.
var workFormByContentKind = map[string]string{
// screen
"movie": "standalone",
"series": "serial",
"episode": "serial",
// music — an album is a closed work, not an ongoing run.
"track": "standalone",
"album": "standalone",
"artist": "standalone",
"musicVideo": "standalone",
"djMix": "standalone",
"liveSet": "standalone",
// literature
"book": "standalone",
"audiobook": "standalone",
"comic": "serial",
"manga": "serial",
"magazine": "serial",
// science
"paper": "standalone",
// ⚠ `pack` is deliberately absent: a season pack is `serial`, an album
// release `standalone`, and only the writer knows which. A pack that
// reaches this sweep without a `workForm` keeps none — inventing one would
// be a guess written permanently into the record.
}
// domainScreenPlan is one record's pending rewrite. Empty strings mean "leave
// this field alone".
type domainScreenPlan struct {
hashID string
domain string
workForm string
}
// MigrateDomainScreen rewrites `domain=film|tv` to `domain=screen` and
// backfills `workForm` from `contentKind` across the whole corpus.
//
// Reads in two batched MGET passes (never a per-record SCAN — a single record
// read costs a full keyspace sweep on this store), then writes only the records
// that actually change.
func (c *Client) MigrateDomainScreen(ctx context.Context) (int, error) {
plans, err := c.planDomainScreen(ctx)
if err != nil {
return 0, err
}
// Writes take the client lock themselves, so they happen after the read
// phase has released it.
fixed := 0
for _, p := range plans {
if p.domain != "" {
if err := c.SetProperty(p.hashID, "domain", p.domain); err != nil {
return fixed, fmt.Errorf("set domain on %s: %w", p.hashID, err)
}
}
if p.workForm != "" {
if err := c.SetProperty(p.hashID, "workForm", p.workForm); err != nil {
return fixed, fmt.Errorf("set workForm on %s: %w", p.hashID, err)
}
}
fixed++
}
return fixed, nil
}
// planDomainScreen is the read half: what would change, without changing it.
func (c *Client) planDomainScreen(ctx context.Context) ([]domainScreenPlan, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
if err != nil {
return nil, fmt.Errorf("smembers index: %w", err)
}
if len(hashIDs) == 0 {
return nil, nil
}
// Three keys per record, in a fixed order so the MGET response decodes by
// position: keys[i*3+n] belongs to hashIDs[i].
const fieldsPerFile = 3
fields := [fieldsPerFile]string{"domain", "workForm", "contentKind"}
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
for _, h := range hashIDs {
prefix := c.buildKeyPrefix(h)
for _, f := range fields {
keys = append(keys, prefix+f)
}
}
// Chunked so one MGET never carries the whole corpus.
const chunk = 3000
values := make([]interface{}, 0, len(keys))
for start := 0; start < len(keys); start += chunk {
end := start + chunk
if end > len(keys) {
end = len(keys)
}
got, err := c.client.MGet(ctx, keys[start:end]...).Result()
if err != nil {
return nil, fmt.Errorf("mget: %w", err)
}
values = append(values, got...)
}
str := func(i int) string {
if i >= len(values) {
return ""
}
s, _ := values[i].(string)
return s
}
plans := make([]domainScreenPlan, 0)
for i, h := range hashIDs {
base := i * fieldsPerFile
domain, workForm, contentKind := str(base), str(base+1), str(base+2)
plan := domainScreenPlan{hashID: h}
if replacement, ok := legacyDomainReplacements[domain]; ok {
plan.domain = replacement
}
// A stored `workForm` is authoritative — a `pack` writer knew something
// the table cannot. Only an absent one is filled in.
if workForm == "" {
if form, ok := workFormByContentKind[contentKind]; ok {
plan.workForm = form
}
}
if plan.domain != "" || plan.workForm != "" {
plans = append(plans, plan)
}
}
return plans, nil
}
// MigrateDomainScreenWithTimeout is the convenience wrapper used by the API
// handler. Generous timeout: the read phase is two batched passes, but the
// write phase is one round-trip per changed field.
func (c *Client) MigrateDomainScreenWithTimeout() (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
return c.MigrateDomainScreen(ctx)
}
internal/storage/domain_screen_migration_test.go
-140
View File
@@ -1,140 +0,0 @@
package storage
import (
"context"
"testing"
)
// seedRecord writes one record's classification fields straight through
// SetProperty, which is what a real writer uses (it maintains the field index
// and the root index the sweep reads from).
func seedRecord(t *testing.T, c *Client, hashID string, fields map[string]string) {
t.Helper()
for k, v := range fields {
if err := c.SetProperty(hashID, k, v); err != nil {
t.Fatalf("seed %s/%s: %v", hashID, k, err)
}
}
}
// The merge (METADATA_KEYS.md §14.17) is paid in the data: `domain` is stored,
// not derived on read, so a record left saying `film` is invisible to a
// `domain:screen` wall. This is the sweep that pays it.
func TestMigrateDomainScreen_RewritesLegacyDomainsAndBackfillsWorkForm(t *testing.T) {
c, _ := newTestClient(t, "")
seedRecord(t, c, "rec-movie", map[string]string{
"domain": "film",
"contentKind": "movie",
})
seedRecord(t, c, "rec-episode", map[string]string{
"domain": "tv",
"contentKind": "episode",
})
// Already current, and already carries both keys: must be left alone.
seedRecord(t, c, "rec-current", map[string]string{
"domain": "screen",
"contentKind": "series",
"workForm": "serial",
})
// A different domain entirely — the sweep must not touch it, but it is
// still owed the `workForm` backfill.
seedRecord(t, c, "rec-book", map[string]string{
"domain": "literature",
"contentKind": "book",
})
fixed, err := c.MigrateDomainScreen(context.Background())
if err != nil {
t.Fatalf("migrate: %v", err)
}
if fixed != 3 {
t.Errorf("fixed = %d, want 3 (the two legacy rows + the book's backfill)", fixed)
}
for _, tc := range []struct {
hashID string
domain string
workForm string
}{
{"rec-movie", "screen", "standalone"},
{"rec-episode", "screen", "serial"},
{"rec-current", "screen", "serial"},
{"rec-book", "literature", "standalone"},
} {
m, err := c.GetMetadataFlat(tc.hashID)
if err != nil {
t.Fatalf("read %s: %v", tc.hashID, err)
}
if m["domain"] != tc.domain {
t.Errorf("%s domain = %q, want %q", tc.hashID, m["domain"], tc.domain)
}
if m["workForm"] != tc.workForm {
t.Errorf("%s workForm = %q, want %q", tc.hashID, m["workForm"], tc.workForm)
}
}
}
// A second pass must find nothing to do — the endpoint is called by hand on
// each box, and "run it twice" must not mean "write the whole corpus twice".
func TestMigrateDomainScreen_IsIdempotent(t *testing.T) {
c, _ := newTestClient(t, "mm:")
seedRecord(t, c, "rec-1", map[string]string{
"domain": "tv",
"contentKind": "episode",
})
if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
t.Fatalf("first pass: %v", err)
}
fixed, err := c.MigrateDomainScreen(context.Background())
if err != nil {
t.Fatalf("second pass: %v", err)
}
if fixed != 0 {
t.Errorf("second pass fixed = %d, want 0", fixed)
}
}
// `pack` is the one kind neither routing table can decide — a season pack is
// `serial`, an album release `standalone`. A pack that reaches the sweep
// without a `workForm` must keep none: inventing one writes a guess into the
// record permanently.
func TestMigrateDomainScreen_NeverGuessesAPacksWorkForm(t *testing.T) {
c, _ := newTestClient(t, "")
seedRecord(t, c, "rec-pack", map[string]string{
"domain": "tv",
"contentKind": "pack",
})
// A writer that DID know keeps what it wrote.
seedRecord(t, c, "rec-album", map[string]string{
"domain": "music",
"contentKind": "pack",
"workForm": "standalone",
})
if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
t.Fatalf("migrate: %v", err)
}
m, err := c.GetMetadataFlat("rec-pack")
if err != nil {
t.Fatalf("read: %v", err)
}
if m["domain"] != "screen" {
t.Errorf("domain = %q, want screen (the domain WAS knowable)", m["domain"])
}
if _, ok := m["workForm"]; ok {
t.Errorf("workForm = %q, want absent", m["workForm"])
}
a, err := c.GetMetadataFlat("rec-album")
if err != nil {
t.Fatalf("read: %v", err)
}
if a["workForm"] != "standalone" {
t.Errorf("workForm = %q, want standalone (writer-supplied, never overwritten)", a["workForm"])
}
}
internal/storage/dual_root_migration.go
-41
View File
@@ -1,41 +0,0 @@
package storage
import (
"context"
"fmt"
)
// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
// legacy storage model: /meta/{hash} write handlers that didn't resolve
// CID → UUID first created a parallel midhash-rooted entry alongside the
// watcher's UUID root.
//
// That pattern is now structurally impossible:
// - writes resolve via ResolveRoot before landing (see handlers), and
// - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
// UUID roots, with the reverse index cid:<cid> → uuid as the single
// resolution path.
//
// The legacy on-disk shape this migration targeted (midhash-rooted entries,
// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
// cut-over wipes Redis, so there is nothing to migrate. The function is kept
// as a no-op so the admin endpoint and its wrappers continue to compile and
// respond.
func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
return 0, nil
}
// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
// handler and the startup hook.
func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
return c.MigrateDualRoots(context.Background())
}
// MigrateDualRootsReport returns a human-readable summary. With the dual-root
// pattern retired (see MigrateDualRoots), this always reports nothing to do.
func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
}
internal/storage/field_index.go
-486
View File
@@ -1,486 +0,0 @@
package storage
import (
"context"
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
// The per-record field-name index.
//
// A record is stored as one flat Redis STRING per leaf
// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
// meta-search can react to one field changing instead of diffing a whole hash
// (see docs/project-architecture/persistent-storage.md, and
// dev/test/suites/flat-keys.bats which asserts records are never hashes).
//
// The cost of that shape was that reading one record meant asking Redis "which
// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
// the byte path for every poster meta-share served, which is how it surfaced.
//
// So we keep the flat keys and index their *names*:
//
// file:<uuid>/__fields__ SET{ "filePath", "sizeByte", "cids/bafk…", … }
//
// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
// This mirrors `file:__index__` (the set of every root) one level down, and the
// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
// from the index rather than scanning per root.
//
// **The set is derived, never authoritative.** The flat keys remain the source
// of truth. A record whose set is missing (anything written before this landed)
// falls back to the old SCAN and populates the set on the way out, so existing
// deployments heal themselves on first touch with no migration and no
// SchemaVersion bump — which matters, because a bump is a flag day that makes
// operators wipe Redis (see schema_version.go).
//
// The consistency argument that makes a derived index safe here is meta-core's
// single-writer rule: leader election publishes an empty `redisUrl` on purpose
// (internal/leader/election.go), so nothing outside this process can write a
// `file:` key behind the index's back.
// FieldsField is the reserved field name holding a record's field-name index.
//
// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
// prefix as the record's own fields. That keeps every existing sweep correct
// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
// The flip side is that every reader which enumerates that prefix has to skip
// it, exactly as they already skip `file:__index__`.
const FieldsField = "__fields__"
// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
const mgetBatch = 1000
// buildFieldsKey constructs the key for a record's field-name index set.
func (c *Client) buildFieldsKey(hashID string) string {
return c.buildKeyPrefix(hashID) + FieldsField
}
// isReservedFieldName reports whether a field name is bookkeeping rather than
// metadata, and so must never be returned to an API caller, exported into a
// snapshot, or folded into the search haystack.
func isReservedFieldName(field string) bool {
return field == FieldsField
}
// readMetadataLocked returns a record's full flat metadata. Caller must hold
// c.mu (either mode).
//
// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
// legacy full-keyspace SCAN, whose result is used to populate the index so the
// next read takes the fast path.
//
// Returns (nil, nil) for a record with no fields — callers treat that as 404.
func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
if len(fields) > 0 {
return c.mgetFieldsLocked(ctx, hashID, fields)
}
// No index yet: a record written before the index existed, or one that
// genuinely has no fields. Fall back to the scan, then backfill.
result, scanned, err := c.scanMetadataLocked(ctx, hashID)
if err != nil {
return nil, err
}
if len(scanned) > 0 {
// Best-effort: a failed backfill costs the next read a scan, it does
// not make the read wrong. Never fail the caller's read for it.
if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
logFieldIndexBackfillFailure(hashID, err)
}
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
//
// A field named in the index whose key is gone reads back as nil and is simply
// skipped: a *stale* index entry is harmless. (A *missing* one would silently
// drop data, which is why every write path maintains the set.)
func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
prefix := c.buildKeyPrefix(hashID)
// Keep the reserved name out of the key list AND out of the result — it is
// a SET, so MGET would error on it in real Redis rather than return nil.
wanted := make([]string, 0, len(fields))
for _, f := range fields {
if !isReservedFieldName(f) {
wanted = append(wanted, f)
}
}
if len(wanted) == 0 {
return nil, nil
}
result := make(map[string]string, len(wanted))
for start := 0; start < len(wanted); start += mgetBatch {
end := start + mgetBatch
if end > len(wanted) {
end = len(wanted)
}
chunk := wanted[start:end]
keys := make([]string, len(chunk))
for i, f := range chunk {
keys[i] = prefix + f
}
values, err := c.client.MGet(ctx, keys...).Result()
if err != nil {
return nil, fmt.Errorf("mget failed: %w", err)
}
for i, v := range values {
if s, ok := v.(string); ok {
result[chunk[i]] = s
}
}
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
// MGET. Retained as the fallback for un-indexed records and as the source for
// the backfill. Returns the metadata plus the field names it saw (which
// includes fields whose value came back nil, so the index stays complete).
func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
prefix := c.buildKeyPrefix(hashID)
result := make(map[string]string)
var fields []string
var cursor uint64
for {
keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, nil, fmt.Errorf("scan failed: %w", err)
}
if len(keys) > 0 {
wanted := make([]string, 0, len(keys))
for _, key := range keys {
field := strings.TrimPrefix(key, prefix)
if isReservedFieldName(field) {
continue
}
fields = append(fields, field)
wanted = append(wanted, key)
}
if len(wanted) > 0 {
values, err := c.client.MGet(ctx, wanted...).Result()
if err != nil {
return nil, nil, fmt.Errorf("mget failed: %w", err)
}
for i, key := range wanted {
if s, ok := values[i].(string); ok {
result[strings.TrimPrefix(key, prefix)] = s
}
}
}
}
cursor = nextCursor
if cursor == 0 {
break
}
}
return result, fields, nil
}
// addFieldsToPipe records field names in the index as part of an existing
// pipeline, so a multi-field write stays one round-trip.
func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
names := filterReserved(fields)
if len(names) == 0 {
return
}
pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
}
// addFieldLocked records a single field name in the index. Caller must hold
// c.mu. Best-effort by the same argument as the backfill: the flat key is the
// source of truth, and a lost index entry is repaired by the next scan
// fallback — so it must not fail the write it accompanies.
func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
if isReservedFieldName(field) {
return
}
if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
logFieldIndexWriteFailure(hashID, field, err)
}
}
// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
//
// A stale entry left behind by a failure is harmless (the MGET reads nil and
// skips it), so this is best-effort too.
func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
if isReservedFieldName(field) {
return
}
if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
logFieldIndexWriteFailure(hashID, field, err)
}
}
// fieldsWithPrefixLocked returns the record's indexed field names under a
// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
// no index yet. Caller must hold c.mu.
func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
if len(fields) == 0 {
// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
// rather than the whole record, and do not backfill from it — a
// partial view must never be mistaken for a complete index.
return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
}
var out []string
for _, f := range fields {
if isReservedFieldName(f) {
continue
}
if strings.HasPrefix(f, fieldPrefix) {
out = append(out, strings.TrimPrefix(f, fieldPrefix))
}
}
// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
// at these sizes and makes the response byte-stable, which matters for
// anything diffing or caching it.
sort.Strings(out)
return out, nil
}
func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
prefix := c.buildKeyPrefix(hashID) + fieldPrefix
var out []string
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, fmt.Errorf("scan fields: %w", err)
}
for _, k := range keys {
out = append(out, strings.TrimPrefix(k, prefix))
}
cursor = next
if cursor == 0 {
break
}
}
sort.Strings(out)
return out, nil
}
// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
//
// Without it the indexes still appear — each record's first read falls back to
// a scan and backfills — but that transition is O(records x keyspace) and it
// runs again from scratch after any wipe. Measured on the real store (12,345
// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
// during which meta-core stays pinned exactly as it was before the fix. Worse,
// `RepublishMetadata` walks every record at startup, so that tail is paid on
// every restart.
//
// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
// for the same reason.
//
// Idempotent, and safe to run against a partially-indexed store: SADD of a
// member that is already there is a no-op. Returns the number of records it
// wrote an index for.
func (c *Client) WarmFieldIndexes() (int, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
indexKey := c.buildIndexKey()
byRecord := make(map[string][]string)
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
if err != nil {
return 0, fmt.Errorf("scan: %w", err)
}
for _, k := range keys {
if k == indexKey {
continue
}
// hashID has no '/'; field may contain them.
rest := strings.TrimPrefix(k, filePrefix)
slash := strings.IndexByte(rest, '/')
if slash < 0 {
continue
}
hashID, field := rest[:slash], rest[slash+1:]
if isReservedFieldName(field) {
// Already indexed — remember the record so we don't treat it
// as missing, but there is nothing to add for this key.
if _, seen := byRecord[hashID]; !seen {
byRecord[hashID] = nil
}
continue
}
byRecord[hashID] = append(byRecord[hashID], field)
}
cursor = next
if cursor == 0 {
break
}
}
written := 0
pipe := c.client.Pipeline()
pending := 0
for hashID, fields := range byRecord {
if len(fields) == 0 {
continue
}
pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
written++
pending++
// Bound the pipeline so a large store doesn't buffer every command.
if pending >= 500 {
if _, err := pipe.Exec(ctx); err != nil {
return written, fmt.Errorf("sadd pipeline: %w", err)
}
pipe = c.client.Pipeline()
pending = 0
}
}
if pending > 0 {
if _, err := pipe.Exec(ctx); err != nil {
return written, fmt.Errorf("sadd pipeline: %w", err)
}
}
return written, nil
}
// recordHasFieldsLocked reports whether a record still has at least one
// property key. Caller must hold c.mu.
//
// `__fields__` must not count — it lives under the record's own prefix, so a
// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
// whose every property has been deleted, and the root could never be
// de-indexed.
func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
if err != nil && err != redis.Nil {
return false, fmt.Errorf("scard fields: %w", err)
}
if n > 0 {
return true, nil
}
// Either genuinely empty, or an un-indexed record from before the index
// existed — the scan is the only way to tell those apart.
prefix := c.buildKeyPrefix(hashID)
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return false, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
return true, nil
}
}
cursor = next
if cursor == 0 {
return false, nil
}
}
}
// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
// record property key. Reports ok=false for anything else — the index set
// itself, `file:__index__`, and any key outside the `file:` namespace.
//
// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
// the index honest without its callers knowing the layout.
func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
rest := key
if c.prefix != "" {
var found bool
rest, found = strings.CutPrefix(rest, c.prefix)
if !found {
return "", "", false
}
}
rest, found := strings.CutPrefix(rest, "file:")
if !found {
return "", "", false
}
hashID, field, found = strings.Cut(rest, "/")
if !found || hashID == "" || field == "" {
// `file:__index__` has no slash and lands here.
return "", "", false
}
if isReservedFieldName(field) {
return "", "", false
}
return hashID, field, true
}
// The index is derived and self-healing, so a write failure degrades
// performance (the next read falls back to a scan and re-populates) rather
// than correctness. Log it rather than failing the caller's write — but log it
// loudly, because a *persistent* failure means every read is scanning again.
func logFieldIndexWriteFailure(hashID, field string, err error) {
log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
}
func logFieldIndexBackfillFailure(hashID string, err error) {
log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
}
func filterReserved(fields []string) []string {
out := make([]string, 0, len(fields))
for _, f := range fields {
if !isReservedFieldName(f) {
out = append(out, f)
}
}
return out
}
func toAny(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i, s := range ss {
out[i] = s
}
return out
}
internal/storage/field_index_test.go
-464
View File
@@ -1,464 +0,0 @@
package storage
import (
"context"
"reflect"
"sort"
"testing"
)
// Tests for the per-record field-name index (field_index.go).
//
// The index is what makes a record read O(fields) instead of O(keyspace). It is
// derived state, so the properties worth pinning are: (1) a read served from
// the index returns exactly what a scan would have, (2) a record with no index
// still reads correctly and gets one, and (3) every write path keeps it in
// sync — because a *missing* entry silently drops a field from every
// subsequent read, which is the one failure mode that would be invisible.
//
// Every case runs with and without a namespace prefix, matching clear_test.go.
func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
t.Helper()
for _, prefix := range []string{"", "mm:"} {
name := "no-prefix"
if prefix != "" {
name = "prefix-" + prefix
}
t.Run(name, func(t *testing.T) {
c, _ := newTestClient(t, prefix)
fn(t, c)
})
}
}
func fieldNames(t *testing.T, c *Client, hashID string) []string {
t.Helper()
got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
if err != nil {
t.Fatalf("smembers fields: %v", err)
}
sort.Strings(got)
return got
}
func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
t.Helper()
got, err := c.GetMetadataFlat(hashID)
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
return got
}
// SetMetadataFlat must index every field it writes, and the read must come back
// through the index rather than a scan.
func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
want := map[string]string{
"filePath": "/files/watch/a.mkv",
"sizeByte": "123",
"cids/bafkreiaaa": "true",
"titles/jpn/ナルト": "true",
"categories/TV/UHD": "true",
"info/files/0/path": "a.mkv",
"provenance": `{"source":"gateway"}`,
"plot/eng": "a plot with / slashes",
"stream/3": "eng",
"genres/action": "true",
"originaltitle": "Naruto",
"deeply/nested/field/name/here": "x",
}
if err := c.SetMetadataFlat("root1", want); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
}
// The index must name every field — including the ones containing '/',
// which is where a naive "split on the first slash" would go wrong.
wantNames := make([]string, 0, len(want))
for k := range want {
wantNames = append(wantNames, k)
}
sort.Strings(wantNames)
if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
}
})
}
// A record written before the index existed has flat keys but no set. The read
// must still be correct, and must leave an index behind so the next read takes
// the fast path. This is the migration path for the ~12k records already on
// disk: no flag day, no SchemaVersion bump.
func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
prefix := c.buildKeyPrefix("legacy")
want := map[string]string{
"filePath": "/files/watch/old.mkv",
"sizeByte": "42",
"cids/bafkreiold": "true",
}
// Write the flat keys directly, bypassing every index-maintaining path
// — exactly the shape a pre-upgrade store is in.
for f, v := range want {
if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
t.Fatalf("seed %s: %v", f, err)
}
}
if names := fieldNames(t, c, "legacy"); len(names) != 0 {
t.Fatalf("precondition: expected no index, got %v", names)
}
if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
}
wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
}
// Second read, now served from the index, must agree with the first.
if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
}
})
}
// Each single-field write path must add to the index, and each delete path must
// remove from it. A field written but not indexed vanishes from every later
// read — silently — so this is the property that matters most.
func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
t.Fatalf("SetProperty: %v", err)
}
if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
t.Fatalf("AddToSet: %v", err)
}
if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
t.Fatalf("MergeMetadataFlat: %v", err)
}
want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
t.Errorf("after writes\n got %v\nwant %v", got, want)
}
// DeleteProperty drops the key and the index entry.
if err := c.DeleteProperty("root2", "title"); err != nil {
t.Fatalf("DeleteProperty: %v", err)
}
// RemoveFromSet deletes the key once its last member is gone.
if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
t.Fatalf("RemoveFromSet: %v", err)
}
want = map[string]string{"year": "2002"}
if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
t.Errorf("after deletes\n got %v\nwant %v", got, want)
}
if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
t.Errorf("index after deletes = %v, want [year]", got)
}
})
}
// The raw KV editor can mint or drop a record property out of band. It is the
// one write path whose caller does not know the record layout, so the storage
// layer has to keep the index honest for it.
func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
t.Fatalf("SetRawValue: %v", err)
}
if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
t.Errorf("index after SetRawValue = %v", got)
}
if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
t.Errorf("read after SetRawValue = %v", got)
}
if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
t.Fatalf("DeleteRawKey: %v", err)
}
if got := fieldNames(t, c, "root3"); len(got) != 0 {
t.Errorf("index after DeleteRawKey = %v, want empty", got)
}
// Keys outside the record namespace must be left alone entirely.
if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
t.Fatalf("SetRawValue non-record: %v", err)
}
if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
t.Error("cid: key was treated as a record property")
}
if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
t.Error("file:__index__ was treated as a record property")
}
})
}
// Mint and AddAlias write fields through their own pipelines, so they need the
// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
// it and a record's CID key-set becomes invisible to every read.
func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
if err != nil {
t.Fatalf("Mint: %v", err)
}
if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
t.Errorf("index after Mint = %v", got)
}
if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
t.Fatalf("AddAlias: %v", err)
}
got := mustGetMetadata(t, c, uuid)
if got["cids/bafkreiaaa"] != "true" {
t.Errorf("alias key-set member missing from read: %v", got)
}
// And the CID enumeration, which now reads the index instead of
// scanning, must see it.
cids, err := c.cidsForRootLocked(context.Background(), uuid)
if err != nil {
t.Fatalf("cidsForRootLocked: %v", err)
}
if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
}
})
}
// The index lives under the record's own prefix, so the "is this record empty?"
// probe would otherwise always find it and the root could never be de-indexed.
func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetProperty("root4", "title", "x"); err != nil {
t.Fatalf("SetProperty: %v", err)
}
removed, err := c.IndexRemoveIfEmpty("root4")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if removed {
t.Error("de-indexed a record that still has a field")
}
if err := c.DeleteProperty("root4", "title"); err != nil {
t.Fatalf("DeleteProperty: %v", err)
}
removed, err = c.IndexRemoveIfEmpty("root4")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if !removed {
t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
}
// ...and the now-empty index key must not be left orphaned.
n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
if err != nil {
t.Fatalf("exists: %v", err)
}
if n != 0 {
t.Error("orphaned field index left behind after de-indexing")
}
})
}
// The same probe must still work for a record that predates the index.
func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
t.Fatalf("seed: %v", err)
}
if err := c.IndexAdd("legacy2"); err != nil {
t.Fatalf("IndexAdd: %v", err)
}
removed, err := c.IndexRemoveIfEmpty("legacy2")
if err != nil {
t.Fatalf("IndexRemoveIfEmpty: %v", err)
}
if removed {
t.Error("de-indexed an un-indexed record that still has a field")
}
})
}
// The reserved name is bookkeeping. It must never reach an API caller as a
// metadata field, and an inbound payload must never be able to overwrite the
// SET with a string — that would break every subsequent read of the record.
func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
err := c.SetMetadataFlat("root5", map[string]string{
"title": "x",
FieldsField: "malicious",
})
if err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
got := mustGetMetadata(t, c, "root5")
if _, present := got[FieldsField]; present {
t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
}
if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
t.Errorf("read = %v, want just title", got)
}
// The index must still be a SET — i.e. not clobbered by the payload.
if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
t.Errorf("index = %v, want [title]", got)
}
if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
t.Error("SetProperty accepted the reserved field name")
}
})
}
// Deleting a record must take its index with it, or a recreated root would
// inherit phantom field names.
func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
if _, err := c.DeleteMetadata("root6"); err != nil {
t.Fatalf("DeleteMetadata: %v", err)
}
if got := fieldNames(t, c, "root6"); len(got) != 0 {
t.Errorf("index survived DeleteMetadata: %v", got)
}
got, err := c.GetMetadataFlat("root6")
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
if got != nil {
t.Errorf("deleted record still reads back: %v", got)
}
})
}
// A stale index entry (name present, key gone) is the benign direction: the
// MGET reads nil and the field is skipped rather than erroring.
func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
// Name a field that has no key behind it.
if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
t.Fatalf("sadd ghost: %v", err)
}
got := mustGetMetadata(t, c, "root7")
if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
t.Errorf("read = %v, want just title", got)
}
})
}
// WarmFieldIndexes is what keeps an upgrade from spending hours reading
// un-indexed records one full-keyspace scan at a time. It must index every
// record in one pass, be idempotent, and leave already-indexed records alone.
func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
ctx := context.Background()
// Two legacy records seeded straight into Redis (no index), plus one
// written through the normal path (already indexed).
legacy := map[string]map[string]string{
"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
"old2": {"filePath": "/b.mkv", "titles/jpn/ナルト": "true", "deep/a/b": "x"},
}
for root, fields := range legacy {
for f, v := range fields {
if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
t.Fatalf("seed: %v", err)
}
}
if err := c.IndexAdd(root); err != nil {
t.Fatalf("IndexAdd: %v", err)
}
}
if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
n, err := c.WarmFieldIndexes()
if err != nil {
t.Fatalf("WarmFieldIndexes: %v", err)
}
if n < 2 {
t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
}
if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
t.Errorf("old1 index = %v", got)
}
if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
t.Errorf("old2 index = %v", got)
}
if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
t.Errorf("fresh index disturbed = %v", got)
}
// Reads must now agree with what the records actually hold.
for root, want := range legacy {
if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
t.Errorf("%s read = %v, want %v", root, got, want)
}
}
// Idempotent: a second pass changes nothing.
if _, err := c.WarmFieldIndexes(); err != nil {
t.Fatalf("second WarmFieldIndexes: %v", err)
}
if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
t.Errorf("old2 index after second warm = %v", got)
}
})
}
// The bulk search read enumerates the whole file: keyspace and MGETs what it
// finds. The index keys are SETs living in that same namespace, so it has to
// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
if err := c.SetMetadataFlat("root8", map[string]string{
"title": "Naruto Shippuden",
"fileName": "naruto.mkv",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
tuples, err := c.GetSearchIndexTuples()
if err != nil {
t.Fatalf("GetSearchIndexTuples: %v", err)
}
var found bool
for _, tu := range tuples {
if tu.HashID != "root8" {
continue
}
found = true
if _, present := tu.Fields[FieldsField]; present {
t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
}
}
if !found {
t.Fatal("record missing from search tuples")
}
})
}
internal/storage/literature_echo_sweep.go
-326
View File
@@ -1,326 +0,0 @@
package storage
import (
"context"
"fmt"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/cid"
)
// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
// identity and position keys are resolved, never copied from a query, and a
// card carries no position keys and no `workcid`.
//
// # Why a sweep
//
// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
// own filter values onto what it returned. The gateway persists whatever passes
// the filter, so the copy became a permanent binding — and stored records never
// pick up the writer fix: a covered `(upstream, query)` never reaches the
// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
// one. So the copies already at rest are removed here, once.
//
// What the copies cost: an `anilistid` on a MangaDex site card claims an
// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
// anchor — the site card then competes with (and can hide) the real AniList
// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
// join and sort like a position inside one.
//
// # What is swept, and what deliberately is not
//
// - Only `domain=literature` + `fileType=card` records. Nothing else is read
// for writing, whatever keys it carries.
// - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
// go from **every** such card: a card is a work, never a position in one,
// so no legitimate writer puts them there.
// - `anilistid` goes only from a **site card** — a card whose own `cids/`
// key-set holds a site-form card locator and no locator of any other form.
// On an anchor card the id *is* the card's identity (its locator is
// `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
// it: which half the id belongs to is not recoverable at rest, and leaving
// a stamp is the safe direction.
// - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
// Matched by rule, and a query-copied stamp is indistinguishable at rest
// from a title-matched one — deleting would drop real bindings with the
// fake ones.
//
// # Contract
//
// Dry-run unless `apply`: the sweep deletes keys, so the default call only
// reports. Idempotent: a swept record plans nothing on the next pass, so the
// endpoint is safe to call repeatedly and on a fresh box.
// literatureEchoAnchorSources are the card-locator sources that name an id-bag
// work (an anchor), never a site.
//
// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
// excludes them and the two rules agree on every locator minted today. They are
// listed so the rule reads on its own here.
var literatureEchoAnchorSources = map[string]bool{
"anilist": true,
"openlibrary": true,
"tmdb": true,
"imdb": true,
"musicbrainz": true,
"mal": true,
}
// literatureCardEchoFields are removed from every literature card carrying
// them — position keys and the work reference, none of which a card may hold.
var literatureCardEchoFields = []string{
"workcid",
"chapterNumber",
"volumeNumber",
"chapterStart",
"chapterEnd",
}
// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
// then `anilistid`, then the echo fields in literatureCardEchoFields order.
var literatureEchoReadFields = append(
[]string{"domain", "fileType", "anilistid"},
literatureCardEchoFields...,
)
// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
// records; ByKey counts fields per key — removed on an apply pass, that would
// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
type LiteratureEchoSweep struct {
Planned int
Fixed int
ByKey map[string]int
}
// literatureEchoPlan is one record's pending deletes, in a fixed order
// (`anilistid` first, then literatureCardEchoFields order).
type literatureEchoPlan struct {
hashID string
fields []string
}
// isSiteFormCardSource is the site-form rule on an already-decoded source: not
// an anchor source, and shaped like a host.
func isSiteFormCardSource(source string) bool {
return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
}
// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
func isSiteFormCardLocator(cidStr string) bool {
source, _, err := cid.DecodeCard(cidStr)
return err == nil && isSiteFormCardSource(source)
}
// isSiteCard reports whether a card's field names prove it a site card: at
// least one site-form `cids/` card locator and no card locator of another form.
// Non-card members (a content CID) say nothing about which work the card is and
// are ignored. An empty field list — a record with no field-name index — is
// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
func isSiteCard(fields []string) bool {
site := false
for _, f := range fields {
member := cidFromKeysetField(f)
if member == "" {
continue
}
source, _, err := cid.DecodeCard(member)
if err != nil {
continue
}
if !isSiteFormCardSource(source) {
return false
}
site = true
}
return site
}
// SweepLiteratureEchoes removes query echoes stored on literature cards (see
// the file comment). With apply=false it only plans.
//
// Reads in batched passes (never a per-record SCAN — a single record read
// costs a full keyspace sweep on this store), then deletes only the planned
// fields, through DeleteProperty so the field-name index and the search-dirty
// set stay consistent.
func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
res := LiteratureEchoSweep{ByKey: map[string]int{}}
plans, err := c.planLiteratureEchoes(ctx)
if err != nil {
return res, err
}
res.Planned = len(plans)
if !apply {
for _, p := range plans {
for _, f := range p.fields {
res.ByKey[f]++
}
}
return res, nil
}
// Writes take the client lock themselves, so they happen after the read
// phase has released it. ByKey counts what was actually deleted, so a
// failure midway reports the true partial state.
for _, p := range plans {
// DeleteProperty carries its own per-call timeout; checking ctx here is
// what makes the wrapper's deadline bound the whole write phase.
if err := ctx.Err(); err != nil {
return res, err
}
for _, f := range p.fields {
if err := c.DeleteProperty(p.hashID, f); err != nil {
return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
}
res.ByKey[f]++
}
res.Fixed++
}
return res, nil
}
// planLiteratureEchoes is the read half: what would be deleted, without
// deleting it.
func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
if err != nil {
return nil, fmt.Errorf("smembers index: %w", err)
}
if len(hashIDs) == 0 {
return nil, nil
}
// Pass 1: the gate and echo fields for every record, in a fixed order so
// the MGET response decodes by position: keys[i*per+n] belongs to
// hashIDs[i].
per := len(literatureEchoReadFields)
keys := make([]string, 0, len(hashIDs)*per)
for _, h := range hashIDs {
prefix := c.buildKeyPrefix(h)
for _, f := range literatureEchoReadFields {
keys = append(keys, prefix+f)
}
}
// Chunked so one MGET never carries the whole corpus.
const chunk = 3000
values := make([]interface{}, 0, len(keys))
for start := 0; start < len(keys); start += chunk {
end := start + chunk
if end > len(keys) {
end = len(keys)
}
got, err := c.client.MGet(ctx, keys[start:end]...).Result()
if err != nil {
return nil, fmt.Errorf("mget: %w", err)
}
values = append(values, got...)
}
// A nil slot is an absent key. A present key holding "" is still an echo
// and still goes, so presence — not the value — decides.
get := func(i int) (string, bool) {
if i >= len(values) || values[i] == nil {
return "", false
}
s, ok := values[i].(string)
return s, ok
}
type candidate struct {
hashID string
hasAnilist bool
echoes []string
}
var cands []candidate
var probe []string // candidates whose anilistid needs their cids/ key-set
for i, h := range hashIDs {
base := i * per
domain, _ := get(base)
fileType, _ := get(base + 1)
if domain != "literature" || fileType != "card" {
continue
}
cand := candidate{hashID: h}
_, cand.hasAnilist = get(base + 2)
for j, f := range literatureCardEchoFields {
if _, ok := get(base + 3 + j); ok {
cand.echoes = append(cand.echoes, f)
}
}
if !cand.hasAnilist && len(cand.echoes) == 0 {
continue
}
if cand.hasAnilist {
probe = append(probe, h)
}
cands = append(cands, cand)
}
// Pass 2: the field-name index of only the cards carrying an `anilistid`
// — the `cids/` members are field NAMES, so the index is where they are
// listed. Pipelined per chunk; still no SCAN.
//
// A card with no index (never touched since before the index existed, and
// missed by the startup WarmFieldIndexes pass) reads as an empty set and
// keeps its `anilistid`: proving it a site card would need exactly the
// per-record SCAN this sweep refuses, and a stamp left in place is the
// safe direction.
siteCard := make(map[string]bool, len(probe))
for start := 0; start < len(probe); start += mgetBatch {
end := start + mgetBatch
if end > len(probe) {
end = len(probe)
}
pipe := c.client.Pipeline()
cmds := make([]*redis.StringSliceCmd, 0, end-start)
for _, h := range probe[start:end] {
cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
}
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields: %w", err)
}
for k, cmd := range cmds {
fields, err := cmd.Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
}
siteCard[probe[start+k]] = isSiteCard(fields)
}
}
plans := make([]literatureEchoPlan, 0, len(cands))
for _, cand := range cands {
var fields []string
if cand.hasAnilist && siteCard[cand.hashID] {
fields = append(fields, "anilistid")
}
fields = append(fields, cand.echoes...)
if len(fields) > 0 {
plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
}
}
return plans, nil
}
// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
// handler. Generous timeout: the read phase is batched, but the write phase is
// one round-trip per deleted field.
func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
return c.SweepLiteratureEchoes(ctx, apply)
}
internal/storage/literature_echo_sweep_test.go
-249
View File
@@ -1,249 +0,0 @@
package storage
import (
"context"
"encoding/base32"
"encoding/binary"
"reflect"
"strings"
"testing"
)
// Pinned from meta-read `src/card_cid.rs` — literals, not round-trips.
const (
literatureAnchorCardCID = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy" // ("anilist", "manga:105778")
literatureSiteCardCID = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
// A non-card CID a card may also carry.
literatureContentCID = "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
)
// mintCardCID encodes a card locator the way meta-feeder-sdk
// `compute_card_cid` does. Asserted against the pinned literals before use.
func mintCardCID(source, id string) string {
digest := binary.AppendUvarint(nil, uint64(len(source)))
digest = append(digest, source...)
digest = append(digest, id...)
wire := []byte{0x01}
wire = binary.AppendUvarint(wire, 0x1007)
wire = append(wire, 0x00)
wire = binary.AppendUvarint(wire, uint64(len(digest)))
wire = append(wire, digest...)
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(wire))
}
func TestIsSiteFormCardLocator(t *testing.T) {
if got := mintCardCID("anilist", "manga:105778"); got != literatureAnchorCardCID {
t.Fatalf("test encoder drifted: %s != %s", got, literatureAnchorCardCID)
}
if got := mintCardCID("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"); got != literatureSiteCardCID {
t.Fatalf("test encoder drifted: %s != %s", got, literatureSiteCardCID)
}
for _, tc := range []struct {
name string
cid string
want bool
}{
{"anilist anchor (golden)", literatureAnchorCardCID, false},
{"mangadex site (golden)", literatureSiteCardCID, true},
{"openlibrary anchor", mintCardCID("openlibrary", "OL45804W"), false},
{"another site", mintCardCID("weebcentral.com", "/series/01J76XY/chainsaw-man"), true},
// Not an anchor source, but not host-shaped either: meta-read reads it
// as an id-bag key, so it must not be treated as a site.
{"dotless unknown source", mintCardCID("suwayomi", "42"), false},
{"content CID", literatureContentCID, false},
{"title key", "title:chainsaw man", false},
} {
if got := isSiteFormCardLocator(tc.cid); got != tc.want {
t.Errorf("%s: isSiteFormCardLocator = %v, want %v", tc.name, got, tc.want)
}
}
}
// seedLiteratureCorpus writes one record per rule the sweep has to honour.
func seedLiteratureCorpus(t *testing.T, c *Client) {
t.Helper()
// A site card carrying every echo: all of it goes.
seedRecord(t, c, "site-card", map[string]string{
"domain": "literature",
"fileType": "card",
"title": "Chainsaw Man",
"cids/" + literatureSiteCardCID: "true",
"anilistid": "105778",
"workcid": literatureSiteCardCID,
"chapterNumber": "12",
"volumeNumber": "2",
"chapterStart": "1",
"chapterEnd": "20",
})
// An anchor card: its anilistid is its identity and stays; the position
// echo and the workcid still go.
seedRecord(t, c, "anchor-card", map[string]string{
"domain": "literature",
"fileType": "card",
"title": "Chainsaw Man",
"cids/" + literatureAnchorCardCID: "true",
"anilistid": "105778",
"workcid": literatureAnchorCardCID,
"chapterNumber": "3",
})
// A site card that is already clean: nothing planned.
seedRecord(t, c, "clean-site-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureSiteCardCID: "true",
})
// Both locator forms: which one the id belongs to is unknowable, so it
// stays.
seedRecord(t, c, "merged-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureSiteCardCID: "true",
"cids/" + literatureAnchorCardCID: "true",
"anilistid": "105778",
})
// No card locator at all: not provably a site card.
seedRecord(t, c, "unlocated-card", map[string]string{
"domain": "literature",
"fileType": "card",
"cids/" + literatureContentCID: "true",
"anilistid": "105778",
})
// A stored empty value is still an echo.
seedRecord(t, c, "blank-echo-card", map[string]string{
"domain": "literature",
"fileType": "card",
"volumeNumber": "",
})
// A chapter is never swept, whatever it carries.
seedRecord(t, c, "chapter", map[string]string{
"domain": "literature",
"fileType": "archive",
"cids/" + literatureContentCID: "true",
"anilistid": "105778",
"workcid": literatureSiteCardCID,
"chapterNumber": "12",
})
// Not literature: out of scope even though it is a card.
seedRecord(t, c, "screen-card", map[string]string{
"domain": "screen",
"fileType": "card",
"workcid": literatureSiteCardCID,
"chapterNumber": "1",
})
}
func TestPlanLiteratureEchoes_ScopesToLiteratureCards(t *testing.T) {
c, _ := newTestClient(t, "")
seedLiteratureCorpus(t, c)
plans, err := c.planLiteratureEchoes(context.Background())
if err != nil {
t.Fatalf("plan: %v", err)
}
got := map[string][]string{}
for _, p := range plans {
got[p.hashID] = p.fields
}
want := map[string][]string{
"site-card": {"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"},
"anchor-card": {"workcid", "chapterNumber"},
"blank-echo-card": {"volumeNumber"},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("plan =\n %v\nwant\n %v", got, want)
}
}
func TestSweepLiteratureEchoes_DryRunThenApply(t *testing.T) {
c, s := newTestClient(t, "mm:")
seedLiteratureCorpus(t, c)
wantByKey := map[string]int{
"anilistid": 1,
"workcid": 2,
"chapterNumber": 2,
"volumeNumber": 2,
"chapterStart": 1,
"chapterEnd": 1,
}
// Dry run: counts, and nothing written.
dry, err := c.SweepLiteratureEchoes(context.Background(), false)
if err != nil {
t.Fatalf("dry run: %v", err)
}
if dry.Planned != 3 || dry.Fixed != 0 {
t.Errorf("dry run planned/fixed = %d/%d, want 3/0", dry.Planned, dry.Fixed)
}
if !reflect.DeepEqual(dry.ByKey, wantByKey) {
t.Errorf("dry run byKey = %v, want %v", dry.ByKey, wantByKey)
}
if m, _ := c.GetMetadataFlat("site-card"); m["anilistid"] != "105778" {
t.Fatalf("dry run deleted anilistid: %v", m)
}
applied, err := c.SweepLiteratureEchoes(context.Background(), true)
if err != nil {
t.Fatalf("apply: %v", err)
}
if applied.Planned != 3 || applied.Fixed != 3 {
t.Errorf("apply planned/fixed = %d/%d, want 3/3", applied.Planned, applied.Fixed)
}
if !reflect.DeepEqual(applied.ByKey, wantByKey) {
t.Errorf("apply byKey = %v, want %v", applied.ByKey, wantByKey)
}
site, err := c.GetMetadataFlat("site-card")
if err != nil {
t.Fatalf("read site-card: %v", err)
}
for _, k := range []string{"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"} {
if _, ok := site[k]; ok {
t.Errorf("site-card still carries %s", k)
}
// The flat key and its field-name index entry must both be gone, or the
// next read would MGET a ghost field.
if s.Exists("mm:file:site-card/" + k) {
t.Errorf("site-card flat key %s survived", k)
}
if member, _ := s.IsMember("mm:file:site-card/"+FieldsField, k); member {
t.Errorf("site-card field index still lists %s", k)
}
}
if site["title"] != "Chainsaw Man" || site["cids/"+literatureSiteCardCID] != "true" {
t.Errorf("site-card lost fields it should keep: %v", site)
}
anchor, _ := c.GetMetadataFlat("anchor-card")
if anchor["anilistid"] != "105778" {
t.Errorf("anchor-card anilistid = %q, want kept", anchor["anilistid"])
}
if _, ok := anchor["workcid"]; ok {
t.Errorf("anchor-card still carries workcid")
}
chapter, _ := c.GetMetadataFlat("chapter")
if chapter["anilistid"] != "105778" || chapter["workcid"] != literatureSiteCardCID || chapter["chapterNumber"] != "12" {
t.Errorf("chapter was touched: %v", chapter)
}
screen, _ := c.GetMetadataFlat("screen-card")
if screen["workcid"] != literatureSiteCardCID {
t.Errorf("screen-card was touched: %v", screen)
}
merged, _ := c.GetMetadataFlat("merged-card")
if merged["anilistid"] != "105778" {
t.Errorf("merged-card anilistid = %q, want kept", merged["anilistid"])
}
// Idempotent: the endpoint is called by hand, and a second call must find
// nothing — with byKey still an empty map, not nil.
again, err := c.SweepLiteratureEchoes(context.Background(), true)
if err != nil {
t.Fatalf("second pass: %v", err)
}
if again.Planned != 0 || again.Fixed != 0 || again.ByKey == nil || len(again.ByKey) != 0 {
t.Errorf("second pass = %+v, want nothing planned and an empty byKey", again)
}
}
internal/storage/schema_version.go
+43 -56
View File
@@ -1,83 +1,70 @@
package storage
import (
"context"
"fmt"
"strconv"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// SchemaVersion is the Redis-layout version this build of meta-core expects.
// SchemaVersion is the key-layout version this build of meta-core expects.
// Distinct from the snapshot manifest version (which is on-disk format);
// this number describes the in-memory Redis schema — primarily, "are roots
// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
// this number describes the record layout — primarily, "are roots UUIDs or
// content-hash tokens, and do reverse-index keys cid:* exist?"
//
// Bump only when a Redis-layout change can't be detected from the data
// itself or healed by the auto-alias hooks. Each bump is a flag day in
// alpha: operators wipe and restart.
// The Redis → SQLite move did not change the record layout (file:<rid>/<field>,
// cid:<cid>), so it did not bump this: the one-time migration copied records
// verbatim.
//
// Bump only when a layout change can't be detected from the data itself or
// healed by the write path. Each bump is a flag day in alpha: operators wipe
// and restart.
const SchemaVersion = 2
// SchemaVersionKey is the sentinel under which we record SchemaVersion in
// Redis. Checked on every meta-core boot.
// SchemaVersionKey is the sentinel under which we record SchemaVersion.
// Checked on every meta-core boot.
const SchemaVersionKey = "meta-core:schema-version"
// EnsureSchemaVersion verifies the Redis store matches the expected schema
// version. Called once at startup, before any other reads/writes.
// EnsureSchemaVersion verifies the store matches the expected schema version.
// Called once at startup, before any other reads/writes.
//
// Behavior:
// - sentinel present and matches expected → OK
// - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
// - sentinel missing AND file:__index__ non-empty → legacy data, refuse
// - sentinel missing AND no roots → fresh store, stamp sentinel
// - sentinel missing AND roots exist → legacy data, refuse
// - sentinel present but mismatched → refuse, point operator at clean:all
//
// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
// The error message names the wipe remediation because in alpha there
// is no automated migration — operators bring up a clean store.
func (c *Client) EnsureSchemaVersion(expected int) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
key := c.buildKey(SchemaVersionKey)
have, err := c.client.Get(ctx, key).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("get schema version: %w", err)
}
if err == redis.Nil {
// Sentinel absent. Fresh store iff there's no file:__index__ membership.
n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
if sErr != nil && sErr != redis.Nil {
return fmt.Errorf("scard file:__index__: %w", sErr)
return c.update(func(tx *kv.Tx) error {
ctx := tx.Context()
have, ok, err := tx.Get(ctx, SchemaVersionKey)
if err != nil {
return fmt.Errorf("get schema version: %w", err)
}
if n > 0 {
return fmt.Errorf(
"redis has %d indexed roots but no %q sentinel — this is alpha "+
"data from before the UUID-rooted schema landed. Run "+
"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
"to wipe and restart, or downgrade meta-core to a compatible build",
n, SchemaVersionKey)
if !ok {
n, err := tx.Count(ctx, rootsNS)
if err != nil {
return fmt.Errorf("count roots: %w", err)
}
if n > 0 {
return fmt.Errorf(
"the store has %d indexed roots but no %q sentinel — this is alpha "+
"data from before the UUID-rooted schema landed. Delete the "+
"database file to wipe and restart, or downgrade meta-core to a compatible build",
n, SchemaVersionKey)
}
return tx.Put(SchemaVersionKey, strconv.Itoa(expected))
}
if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
return fmt.Errorf("set schema version: %w", err)
if have == strconv.Itoa(expected) {
return nil
}
return nil
}
if have == strconv.Itoa(expected) {
return nil
}
return fmt.Errorf(
"redis schema version is %q, this build of meta-core expects %d. "+
"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
"build that matches the existing data",
have, expected)
return fmt.Errorf(
"store schema version is %q, this build of meta-core expects %d. "+
"Delete the database file to wipe and restart, or use a meta-core "+
"build that matches the existing data",
have, expected)
})
}
internal/storage/search.go
+565
View File
@@ -0,0 +1,565 @@
package storage
// Search over the KV store.
//
// The Redis era answered POST /api/metadata/search from an in-memory copy of
// every record (the API's searchIndex), because Redis could not filter
// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
// gone: search now runs against two derived structures inside the database.
//
// - kv_field_v — an expression index over (lowercased field name parsed out
// of the key, lowercased value, value), covering only record keys. It answers "which
// records have field F with a value like V" for ANY field, including ones
// invented tomorrow: the data stays schema-free, the index is over the key
// itself.
// - fts — an FTS5 trigram table holding one row per record: the record's
// haystack (buildHaystack) plus its rid. Trigrams make every substring of
// 3+ characters indexable, which is the matching rule search always had.
//
// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
// semantics are bit-identical to the in-memory matcher and nothing depends on
// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.
import (
"context"
"encoding/json"
"fmt"
"os"
"sort"
"strconv"
"strings"
"unicode/utf8"
"github.com/metazla/meta-core/internal/kv"
)
// The field expression and the partial-index predicate MUST appear verbatim
// (as literals, never bound parameters) in every query meant to use kv_field_v —
// SQLite only matches a partial index whose WHERE clause it can see implied by
// the query text. TestSearchFilterUsesFieldIndex pins this.
const (
fieldExpr = `lower(substr(k, instr(k,'/')+1))`
recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
ridExpr = `substr(k, 6, instr(k,'/')-6)`
)
// searchSchema is the derived DDL applied when the database is opened.
//
// kv_field_v carries the raw value as a third column so the filters' substring
// test (instr(lower(v), …)) is answered from the index alone. Without it,
// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
// lower(v)), which is dropped here; the rebuild costs ~80 s once.
const searchSchema = `
DROP INDEX IF EXISTS kv_field;
CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
`
// searchIndexFields are the metadata fields folded into a record's search
// haystack — must stay in sync with the fields the search handler documents.
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
// affectsHaystack reports whether writing field can change the haystack.
func affectsHaystack(field string) bool {
if strings.HasPrefix(field, "titles/") {
return true
}
for _, f := range searchIndexFields {
if field == f {
return true
}
}
return false
}
// Haystack folds a record's searchable text fields into one lowercased blob —
// what free-text tokens must each be a substring of.
//
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
// record whose `title` is `3%`.
func Haystack(fields map[string]string) string {
var sb strings.Builder
for _, f := range searchIndexFields {
if v, ok := fields[f]; ok && v != "" {
sb.WriteString(strings.ToLower(v))
sb.WriteByte('\n')
}
}
for _, name := range titleMemberNames(fields) {
sb.WriteString(strings.ToLower(name))
sb.WriteByte('\n')
}
return sb.String()
}
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
// back. Sorted so the haystack is deterministic.
func titleMemberNames(fields map[string]string) []string {
var names []string
for k, v := range fields {
rest, ok := strings.CutPrefix(k, "titles/")
if !ok || v == "" || v == "false" {
continue
}
_, name, ok := strings.Cut(rest, "/")
if !ok || name == "" {
continue
}
names = append(names, strings.ReplaceAll(name, "∕", "/"))
}
sort.Strings(names)
return names
}
// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
// "query is a substring of the hashID" rule also has an indexed candidate path.
// Verification checks the two separately (a rid token plus a title token must
// not match together).
func ftsHay(hashID string, fields map[string]string) string {
return Haystack(fields) + strings.ToLower(hashID) + "\n"
}
// refreshSearchTx rewrites a record's FTS row from its current fields, or
// drops it when the record is gone.
func refreshSearchTx(tx *kv.Tx, hashID string) error {
fields, err := readRecord(tx.Context(), tx, hashID)
if err != nil {
return err
}
if len(fields) == 0 {
return deleteSearchRowTx(tx, hashID)
}
return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
}
// ensureSearchRowTx creates the record's FTS row if it has none (a record
// whose writes never touched a haystack field still needs one for the rid
// rule).
func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil || ok {
return err
}
return refreshSearchTx(tx, hashID)
}
func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil {
return err
}
if ok {
if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
return fmt.Errorf("fts update %s: %w", hashID, err)
}
return nil
}
res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
if err != nil {
return fmt.Errorf("fts insert %s: %w", hashID, err)
}
rowid, err := res.LastInsertId()
if err != nil {
return err
}
return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
}
func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
if err != nil || !ok {
return err
}
if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
return fmt.Errorf("fts delete %s: %w", hashID, err)
}
_, err = tx.Delete(ftsNS + hashID)
return err
}
// SearchFilter is one structured filter: AND across filters, OR across Values.
type SearchFilter struct {
Property string
Values []string
}
// SearchHit is one verified result.
type SearchHit struct {
HashID string
Fields map[string]string
}
// Search returns up to limit records, in hashID order, that satisfy match.
//
// query and filters only NARROW the candidates (SQL, superset semantics);
// match is the authoritative test, run on each hydrated record. Pass the same
// query/filters match evaluates, or results will be missing.
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
s, err := c.kv()
if err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
ctx, cancel := opCtx()
defer cancel()
driver, probes, err := c.planCandidates(ctx, s, query, filters)
if err != nil {
return nil, err
}
page := max(limit*2, 500)
hits := make([]SearchHit, 0, limit)
after := ""
for len(hits) < limit {
ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
if err != nil {
return nil, err
}
survivors := ids
if len(probes) > 0 && len(ids) > 0 {
if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
return nil, err
}
}
for _, id := range survivors {
fields, err := readRecord(ctx, s, id)
if err != nil {
return nil, err
}
if fields == nil || !match(id, fields) {
continue
}
hits = append(hits, SearchHit{HashID: id, Fields: fields})
if len(hits) >= limit {
break
}
}
if len(ids) < page {
break
}
after = ids[len(ids)-1]
}
return hits, nil
}
// queryIDs returns one page of the driver's candidates, in rid order.
func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
if err != nil {
return nil, fmt.Errorf("search candidates: %w", err)
}
defer rows.Close()
var out []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
out = append(out, id)
}
return out, rows.Err()
}
// probeSQL applies the non-driving filters to one page of candidate rids
// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
func probeSQL(probes []candidatePart) (string, []any) {
where := make([]string, len(probes))
var args []any
for i, p := range probes {
where[i] = p.exists
args = append(args, p.existsArgs...)
}
return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
}
// probeIDs keeps the ids (in their order) that pass every probe.
func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
q, args := probeSQL(probes)
list, err := json.Marshal(ids)
if err != nil {
return nil, err
}
rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
if err != nil {
return nil, fmt.Errorf("search probes: %w", err)
}
defer rows.Close()
pass := make(map[string]bool, len(ids))
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
pass[id] = true
}
if err := rows.Err(); err != nil {
return nil, err
}
out := ids[:0:0]
for _, id := range ids {
if pass[id] {
out = append(out, id)
}
}
return out, nil
}
// candidatePart is one narrowing condition: free text, or one filter.
type candidatePart struct {
sql string // SELECT … AS rid
args []any
// exists is the same filter as a predicate on candidate row c.rid (empty
// for free text, which has no per-record probe: the FTS rid column is not
// indexed).
exists string
existsArgs []any
}
// estimateCap bounds the count used to pick the driving part: it only has to
// tell a 50-row part from a 100k-row one.
const estimateCap = 1000
// planCandidates picks how to narrow a search: one DRIVING part (the most
// selective, by a capped count) that lists candidate rids in order, plus PROBES
// — every other filter, checked as an EXISTS over each candidate's own key
// range (a primary-key range read of ~40 rows).
//
// Search pages the driver in Go and probes one page at a time, so a broad
// query stops as soon as `limit` records verify. Two earlier shapes were
// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
// and probing inside the driver query (SQLite evaluates the probe on every
// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
// both ~100k, 10 s). Free text that does not drive is dropped from SQL
// entirely — the verifier still applies it, and dropping a condition only
// widens the superset.
func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
var parts []candidatePart
if p, ok := textCandidates(query); ok {
parts = append(parts, p)
}
for _, f := range filters {
if p, ok := filterCandidates(f); ok {
parts = append(parts, p)
}
}
if len(parts) == 0 {
return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
}
driver := 0
if len(parts) > 1 {
best := int64(-1)
for i, p := range parts {
var n int64
q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
}
if best < 0 || n < best {
best, driver = n, i
}
}
}
var probes []candidatePart
for i, p := range parts {
if i != driver && p.exists != "" {
probes = append(probes, p)
}
}
return parts[driver], probes, nil
}
// candidateSQL is planCandidates without the store: text plus filters, the
// first part driving. Used by tests that inspect the plan.
func candidateSQL(query string, filters []SearchFilter) (string, []any) {
var parts []candidatePart
if p, ok := textCandidates(query); ok {
parts = append(parts, p)
}
for _, f := range filters {
if p, ok := filterCandidates(f); ok {
parts = append(parts, p)
}
}
if len(parts) == 0 {
return "", nil
}
return parts[0].sql, parts[0].args
}
// textCandidates narrows by free text: every whitespace token must be a
// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
// same rows.
func textCandidates(query string) (candidatePart, bool) {
tokens := strings.Fields(strings.ToLower(query))
if len(tokens) == 0 {
return candidatePart{}, false
}
var match []string
var likes []string
var likeArgs []any
for _, t := range tokens {
if utf8.RuneCountInString(t) >= 3 {
match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
} else {
likes = append(likes, `hay LIKE ? ESCAPE '\'`)
likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
}
}
var where []string
var args []any
if len(match) > 0 {
where = append(where, `fts MATCH ?`)
args = append(args, strings.Join(match, " "))
}
where = append(where, likes...)
args = append(args, likeArgs...)
return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
}
func escapeLike(s string) string {
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
return r.Replace(s)
}
// filterCandidates narrows by one filter — the UNION, over its values, of the
// two storage shapes recordMatchesFilter probes:
//
// - key-set member `<prop>/<value>` (any case): the lowercased field equals
// lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
// lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
// comparison could miss);
// - flat field `<prop>` whose lowercased value contains the lowercased value
// — covers exact and contains alike, and comma-split csv-set values.
//
// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
// name): it constrains nothing in SQL and is left to the verifier.
func filterCandidates(f SearchFilter) (candidatePart, bool) {
var values []string
for _, v := range f.Values {
if v = strings.TrimSpace(v); v != "" {
values = append(values, v)
}
}
if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
return candidatePart{}, false
}
prop := asciiLower(f.Property)
var conds []string
var args []any
memberRange := false
for _, v := range values {
if isASCII(v) {
lv := asciiLower(v)
conds = append(conds, fieldExpr+` = ?`)
args = append(args, prop+"/"+lv)
conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
args = append(args, prop, lv)
} else {
memberRange = true
conds = append(conds, fieldExpr+` = ?`)
args = append(args, prop)
}
}
if memberRange {
conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
}
// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
// OR inside one SELECT would defeat the partial index; plain UNION would
// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
selects := make([]string, len(conds))
for i, cond := range conds {
selects[i] = base + cond
}
// As a probe: the same conditions over one candidate's key range.
exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
strings.Join(conds, ") OR (") + `)))`
return candidatePart{
sql: strings.Join(selects, " UNION ALL "), args: args,
exists: exists, existsArgs: args,
}, true
}
func isASCII(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] >= utf8.RuneSelf {
return false
}
}
return true
}
// asciiLower lowercases A–Z only, exactly like SQLite's lower().
func asciiLower(s string) string {
b := []byte(s)
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + ('a' - 'A')
}
}
return string(b)
}
func envInt(name string, def int) int {
if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
return v
}
return def
}
// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
// empty but records exist — a store restored from a snapshot, a copy made
// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
// change its tokenizer. The rows are derived, so this is a repair, not a
// migration. Runs in bounded batches so writes keep flowing; until it
// finishes, free-text search under-matches (filters are unaffected).
func (c *Client) RebuildSearchIfEmpty() (int, error) {
s, err := c.kv()
if err != nil {
return 0, err
}
ctx, cancel := opCtx()
var rows int64
err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
cancel()
if err != nil {
return 0, fmt.Errorf("count fts rows: %w", err)
}
if rows > 0 {
return 0, nil
}
ids, err := c.GetAllHashIDs()
if err != nil {
return 0, err
}
const batch = 500
for start := 0; start < len(ids); start += batch {
chunk := ids[start:min(start+batch, len(ids))]
if err := c.update(func(tx *kv.Tx) error {
for _, id := range chunk {
if err := refreshSearchTx(tx, id); err != nil {
return err
}
}
return nil
}); err != nil {
return start, fmt.Errorf("rebuild search rows: %w", err)
}
}
return len(ids), nil
}
internal/storage/search_dirty.go
-245
View File
@@ -1,245 +0,0 @@
package storage
import (
"context"
"fmt"
"log"
"sort"
"strings"
"time"
)
// Incremental search-index maintenance.
//
// # Why this exists
//
// The API's in-memory search index used to be rebuilt in full every 60 s:
// GetSearchIndexTuples does a whole-keyspace SCAN plus batched MGETs, builds an
// intermediate map of every record, then allocates a fresh []SearchTuple — all
// while the previous index is still live serving reads. That cost is O(corpus),
// not O(change): adding 200 records in a minute re-read every one of the other
// ~620 000 field keys to notice them.
//
// Measured on a 19 622-record staging corpus (2026-08-30): SCAN was 57.8 % and
// MGET 19.1 % of ALL Redis CPU, and ~90 % of the MGET calls plus ~48 % of the
// SCAN calls were this loop — roughly 45 % of Redis's total work, spent
// continuously whether or not anything changed. The Go side held 783 MB for
// data Redis stores in 135 MB, largely because of the double copy.
//
// That was tolerable while the corpus was small and mostly static. It stops
// being tolerable now that meta-core is the platform's permanent, additive
// store: every search hit on every gateway is retained forever, so the rebuild
// cost is a ratchet.
//
// # The shape
//
// Every write path marks its record in a dirty set; the refresh drains that set
// and re-reads only those records. Cost becomes O(change).
//
// Marking rides along with the field-index maintenance every write path already
// performs, so there is no new bookkeeping discipline to remember — if you are
// adding a write path, the same line that keeps `__fields__` correct keeps this
// correct.
//
// # Correctness
//
// The drain is a destructive SPOP, so a write that lands *during* a refresh is
// either already in the drained batch or still in the set for the next tick —
// never dropped. The cost of the race is one tick of staleness, which is well
// inside the interval the index was already stale by.
//
// A refresh that fails after draining WOULD lose its batch, so drainDirty's
// caller re-marks on error (see MarkDirty / the API's refreshIncremental).
//
// ⚠ **The destructive drain assumes ONE index consumer per Redis.** Two
// meta-core processes sharing a keyspace would steal each other's marks, and
// each would end up with an index missing whatever the other drained. That is
// fine today — multi-instance leader election was deprecated and the dev/prod
// stacks run a single meta-core — but it is the assumption to revisit first if
// a second instance is ever pointed at the same Redis. A per-consumer dirty key
// (`file:__dirty__:<consumer>`) is the shape that would fix it.
//
// The API's periodic full reconcile (searchIndexReconcileEvery) is the backstop
// for both this and for a write path that forgets to mark: a missed mark costs
// staleness until the next reconcile, never permanent divergence.
// dirtyField is the reserved key holding the set of record ids whose metadata
// changed since the last search-index refresh. It sits under the same `file:`
// namespace as `__index__` and is likewise not a record.
const dirtyField = "file:__dirty__"
// dirtyDrainCap bounds one incremental refresh. A burst larger than this is
// drained over several ticks rather than in one long-held pass — the point of
// the change is to keep any single refresh cheap. Sized well above a normal
// search-persist flush (which writes at most a few hundred records).
const dirtyDrainCap = 5000
// buildDirtyKey constructs the key for the dirty-record set.
func (c *Client) buildDirtyKey() string {
return c.buildKey(dirtyField)
}
// markDirtyLocked records that hashID's metadata changed. Caller must hold c.mu.
//
// Best-effort by design: a failure here costs one record its incremental
// refresh, and the periodic full rebuild still repairs it. Failing the caller's
// write instead would trade a stale index entry for a lost write, which is
// strictly worse.
func (c *Client) markDirtyLocked(ctx context.Context, hashID string) {
if hashID == "" {
return
}
if err := c.client.SAdd(ctx, c.buildDirtyKey(), hashID).Err(); err != nil {
log.Printf("[STORAGE] dirty-mark failed for %s: %v (index repairs on the next full rebuild)", hashID, err)
}
}
// MarkDirty records that hashID needs re-indexing. Exported for the API layer,
// which re-marks a batch whose refresh failed after the drain.
func (c *Client) MarkDirty(hashIDs ...string) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil || len(hashIDs) == 0 {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
members := make([]interface{}, 0, len(hashIDs))
for _, h := range hashIDs {
if h != "" {
members = append(members, h)
}
}
if len(members) == 0 {
return
}
if err := c.client.SAdd(ctx, c.buildDirtyKey(), members...).Err(); err != nil {
log.Printf("[STORAGE] dirty re-mark failed for %d records: %v", len(members), err)
}
}
// DrainDirty removes and returns up to dirtyDrainCap record ids from the dirty
// set. `more` reports whether entries remain, so the caller can schedule
// another pass instead of waiting a full interval to catch up on a burst.
//
// Destructive on purpose: SPOP makes concurrent refreshes disjoint and stops a
// record being re-read every tick forever. The caller owns the drained ids
// until it has indexed them — on failure it must MarkDirty them again.
func (c *Client) DrainDirty() (ids []string, more bool, err error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
key := c.buildDirtyKey()
ids, err = c.client.SPopN(ctx, key, dirtyDrainCap).Result()
if err != nil {
return nil, false, fmt.Errorf("spop dirty: %w", err)
}
if len(ids) < dirtyDrainCap {
return ids, false, nil
}
remaining, err := c.client.SCard(ctx, key).Result()
if err != nil {
// We have the batch; not knowing the remainder just means the next
// tick picks it up on schedule.
return ids, false, nil
}
return ids, remaining > 0, nil
}
// GetSearchIndexTuplesFor reads the search tuples for a specific set of records
// — the incremental counterpart to GetSearchIndexTuples.
//
// Reads go through the per-record field index (GetMetadataFlat), so this is two
// round-trips per record rather than a keyspace scan. That trade is what makes
// it O(change): it beats the full pass whenever the changed set is a small
// fraction of the corpus, and the caller bounds the batch (dirtyDrainCap).
//
// A record present in the dirty set but absent from storage was deleted; it is
// reported in `gone` so the caller can evict it from the index rather than
// leaving a tombstone that still answers searches.
func (c *Client) GetSearchIndexTuplesFor(hashIDs []string) (tuples []SearchTuple, gone []string, err error) {
if len(hashIDs) == 0 {
return nil, nil, nil
}
excluded := indexExcludePrefixes()
tuples = make([]SearchTuple, 0, len(hashIDs))
for _, hashID := range hashIDs {
fields, ferr := c.GetMetadataFlat(hashID)
if ferr != nil {
// Re-mark so the record is retried rather than silently missing
// from the index until the next full rebuild.
c.MarkDirty(hashID)
log.Printf("[STORAGE] incremental index read failed for %s: %v", hashID, ferr)
continue
}
if len(fields) == 0 {
gone = append(gone, hashID)
continue
}
for field := range fields {
if hasAnyPrefix(field, excluded) {
delete(fields, field)
}
}
tuples = append(tuples, SearchTuple{
HashID: hashID,
Haystack: buildHaystack(fields),
Fields: fields,
})
}
return tuples, gone, nil
}
// buildHaystack folds a record's searchable text fields into one lowercased
// blob. Shared by the full and incremental index reads so the two can never
// disagree about what is searchable — a drift there would make a record's
// matchability depend on which path happened to index it.
//
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
// record whose `title` is `3%`.
func buildHaystack(fields map[string]string) string {
var sb strings.Builder
for _, f := range searchIndexFields {
if v, ok := fields[f]; ok && v != "" {
sb.WriteString(strings.ToLower(v))
sb.WriteByte('\n')
}
}
for _, name := range titleMemberNames(fields) {
sb.WriteString(strings.ToLower(name))
sb.WriteByte('\n')
}
return sb.String()
}
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
// back. Sorted: the fields map has no order, and the full and incremental
// builds must produce byte-identical haystacks.
func titleMemberNames(fields map[string]string) []string {
var names []string
for k, v := range fields {
rest, ok := strings.CutPrefix(k, "titles/")
if !ok || v == "" || v == "false" {
continue
}
_, name, ok := strings.Cut(rest, "/")
if !ok || name == "" {
continue
}
names = append(names, strings.ReplaceAll(name, "\u2215", "/"))
}
sort.Strings(names)
return names
}
internal/storage/search_dirty_test.go
-230
View File
@@ -1,230 +0,0 @@
package storage
import (
"context"
"sort"
"testing"
)
// Tests for incremental search-index maintenance (search_dirty.go).
//
// The dirty set is derived state on the hot write path, which makes its failure
// mode the dangerous kind: a MISSING mark doesn't error, it just leaves the
// search index quietly serving a stale copy of that record until someone
// triggers a full rebuild. So the property worth pinning is coverage — every
// path that changes what a search would match must mark — and that is what the
// bulk of these tests assert, one write path at a time.
//
// The second property is that a delete is a mark too, not an unmark: the
// refresh has to LEARN the record is gone in order to evict it. Marking on
// delete is counter-intuitive enough to be worth a test of its own.
func dirtyMembers(t *testing.T, c *Client) []string {
t.Helper()
got, err := c.client.SMembers(context.Background(), c.buildDirtyKey()).Result()
if err != nil {
t.Fatalf("smembers dirty: %v", err)
}
sort.Strings(got)
return got
}
func clearDirty(t *testing.T, c *Client) {
t.Helper()
if err := c.client.Del(context.Background(), c.buildDirtyKey()).Err(); err != nil {
t.Fatalf("del dirty: %v", err)
}
}
// Every write path must mark. A path that doesn't leaves the index serving a
// stale record with no error anywhere.
func TestDirty_EveryWritePathMarks(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
cases := []struct {
name string
id string
write func(id string) error
}{
{"SetMetadataFlat", "r-set", func(id string) error {
return c.SetMetadataFlat(id, map[string]string{"title": "A"})
}},
{"SetProperty", "r-prop", func(id string) error {
return c.SetProperty(id, "title", "B")
}},
{"MergeMetadataFlat", "r-merge", func(id string) error {
_, err := c.MergeMetadataFlat(id, map[string]string{"title": "C"})
return err
}},
{"AddToSet", "r-add", func(id string) error {
_, err := c.AddToSet(id, "tags", "x")
return err
}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
clearDirty(t, c)
if err := tc.write(tc.id); err != nil {
t.Fatalf("%s: %v", tc.name, err)
}
got := dirtyMembers(t, c)
if len(got) != 1 || got[0] != tc.id {
t.Fatalf("%s must mark %q dirty, got %v", tc.name, tc.id, got)
}
})
}
})
}
// RemoveFromSet's non-empty branch changes a field's VALUE without changing the
// field-name index, so it is the one path that a mark riding on field-index
// maintenance would have missed. Pinned explicitly for that reason.
func TestDirty_RemoveFromSetMarksEvenWhenTheFieldSurvives(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-rem"
for _, v := range []string{"x", "y"} {
if _, err := c.AddToSet(id, "tags", v); err != nil {
t.Fatalf("AddToSet: %v", err)
}
}
clearDirty(t, c)
if _, err := c.RemoveFromSet(id, "tags", "x"); err != nil {
t.Fatalf("RemoveFromSet: %v", err)
}
// `tags` still exists (y remains) — no field-index change, but the
// searchable value changed.
if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
t.Fatalf("a value-only change must still mark dirty, got %v", got)
}
})
}
// A delete MARKS rather than unmarks: the refresh must read the record, find it
// gone, and evict it. Unmarking would leave a deleted record answering searches
// until the next full rebuild.
func TestDirty_DeletesMarkSoTheRefreshCanEvict(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-del"
if err := c.SetMetadataFlat(id, map[string]string{"title": "Doomed"}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
clearDirty(t, c)
if _, err := c.DeleteMetadata(id); err != nil {
t.Fatalf("DeleteMetadata: %v", err)
}
if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
t.Fatalf("a delete must mark dirty so the index evicts it, got %v", got)
}
// …and the incremental read reports it as gone rather than as an error
// or an empty-but-present tuple.
tuples, gone, err := c.GetSearchIndexTuplesFor([]string{id})
if err != nil {
t.Fatalf("GetSearchIndexTuplesFor: %v", err)
}
if len(tuples) != 0 {
t.Fatalf("a deleted record must produce no tuple, got %d", len(tuples))
}
if len(gone) != 1 || gone[0] != id {
t.Fatalf("a deleted record must be reported gone, got %v", gone)
}
})
}
// The drain is destructive so concurrent refreshes stay disjoint and a record
// isn't re-read every tick forever. The caller owns what it drained.
func TestDirty_DrainIsDestructive(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
clearDirty(t, c)
c.MarkDirty("a", "b", "c")
ids, more, err := c.DrainDirty()
if err != nil {
t.Fatalf("DrainDirty: %v", err)
}
sort.Strings(ids)
if len(ids) != 3 {
t.Fatalf("expected 3 drained ids, got %v", ids)
}
if more {
t.Fatal("a fully-drained set must not report more")
}
if got := dirtyMembers(t, c); len(got) != 0 {
t.Fatalf("drain must empty the set, got %v", got)
}
// A second drain is empty, not an error — the steady state when nothing
// has been written since the last tick.
ids, _, err = c.DrainDirty()
if err != nil || len(ids) != 0 {
t.Fatalf("draining an empty set must be a clean no-op, got %v / %v", ids, err)
}
})
}
// The incremental read must produce tuples indistinguishable from the full
// rebuild's — same haystack, same fields, same exclusions. A drift here would
// make a record's matchability depend on which path happened to index it, which
// is invisible until someone searches for the one term that differs.
func TestDirty_IncrementalTuplesMatchTheFullRebuild(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "r-parity"
if err := c.SetMetadataFlat(id, map[string]string{
"title": "Parity Release",
"fileName": "parity.s01e01.mkv",
"fileType": "video",
"titles/eng/3 Percent": "true",
"titles/fra/3 Pourcent": "true",
"categories/XXX": "true",
"categories/newznab/5070": "true",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
full := tuplesByHashID(t, c)[id]
incr, gone, err := c.GetSearchIndexTuplesFor([]string{id})
if err != nil {
t.Fatalf("GetSearchIndexTuplesFor: %v", err)
}
if len(gone) != 0 || len(incr) != 1 {
t.Fatalf("expected exactly one live tuple, got %d live / %d gone", len(incr), len(gone))
}
if incr[0].Haystack != full.Haystack {
t.Errorf("haystack drift:\n incremental %q\n full %q", incr[0].Haystack, full.Haystack)
}
if len(incr[0].Fields) != len(full.Fields) {
t.Errorf("field-count drift: incremental %d, full %d", len(incr[0].Fields), len(full.Fields))
}
for k, v := range full.Fields {
if incr[0].Fields[k] != v {
t.Errorf("field %q drift: incremental %q, full %q", k, incr[0].Fields[k], v)
}
}
// The index projection must apply on both paths.
if _, ok := incr[0].Fields["categories/newznab/5070"]; ok {
t.Error("the incremental path must honour the index exclusion list too")
}
if _, ok := incr[0].Fields["categories/XXX"]; !ok {
t.Error("the incremental path must keep the adult-filter category")
}
})
}
// A `titles/<lang3>/<name>` member contributes its NAME (the key), never its
// "true" value, and the `∕` a writer substituted for `/` reads back as `/`.
func TestBuildHaystack_FoldsTitleMemberNames(t *testing.T) {
got := buildHaystack(map[string]string{
"title": "3%",
"titles/fra/3 Pourcent": "true",
"titles/eng/3 Percent": "true",
"titles/und/Fate\u2215Zero": "true",
"titles/eng": "Retired Scalar",
"titles/eng/": "true",
})
want := "3%\n3 percent\n3 pourcent\nfate/zero\n"
if got != want {
t.Errorf("haystack = %q, want %q", got, want)
}
}
internal/storage/search_index_exclude_test.go
-116
View File
@@ -1,116 +0,0 @@
package storage
import (
"testing"
)
// Tests for the search-index field projection (GetSearchIndexTuples +
// indexExcludePrefixes).
//
// The projection narrows the in-memory index only — Redis keeps every field and
// every per-record read still returns it. What has to be pinned is the boundary
// between the two, because getting it wrong in either direction is silent:
// exclude too little and the index keeps paying for fields nothing reads;
// exclude too much and a record served *from the index* comes back missing a
// field a downstream consumer depends on.
//
// The specific hazard the default guards is adult-content filtering:
// `categories/XXX` is the server-side enforcement point (meta-search's
// `query_eval::fields_are_adult`, meta-gateway's `cards::fields_are_adult`),
// and `exclude_adult` defaults to ON. Excluding the whole `categories/` family
// would make every index-served record read as non-adult and the filter would
// fail OPEN — so the default must keep `categories/XXX` while dropping the
// reader-less `categories/newznab/` mirror.
func tuplesByHashID(t *testing.T, c *Client) map[string]SearchTuple {
t.Helper()
got, err := c.GetSearchIndexTuples()
if err != nil {
t.Fatalf("GetSearchIndexTuples: %v", err)
}
out := make(map[string]SearchTuple, len(got))
for _, tu := range got {
out[tu.HashID] = tu
}
return out
}
func TestSearchIndex_ExcludesNewznabCategoriesButKeepsTheRest(t *testing.T) {
forEachPrefix(t, func(t *testing.T, c *Client) {
const id = "root-1"
if err := c.SetMetadataFlat(id, map[string]string{
"title": "Some Release",
"fileType": "video",
"categories/TV/Anime": "true",
"categories/XXX": "true",
"categories/newznab/5070": "true",
"categories/newznab/2040": "true",
}); err != nil {
t.Fatalf("SetMetadataFlat: %v", err)
}
fields := tuplesByHashID(t, c)[id].Fields
if fields == nil {
t.Fatal("record missing from the search index")
}
for _, excluded := range []string{"categories/newznab/5070", "categories/newznab/2040"} {
if _, ok := fields[excluded]; ok {
t.Errorf("%s must not be indexed — it is a reader-less mirror vocabulary", excluded)
}
}
// The load-bearing ones must survive. XXX especially: without it the
// adult filter fails open on every index-served record.
for _, kept := range []string{"categories/XXX", "categories/TV/Anime", "title", "fileType"} {
if _, ok := fields[kept]; !ok {
t.Errorf("%s must stay in the index", kept)
}
}
// The store itself is untouched — the projection is index-only.
stored, err := c.GetMetadataFlat(id)
if err != nil {
t.Fatalf("GetMetadataFlat: %v", err)
}
if _, ok := stored["categories/newznab/5070"]; !ok {
t.Error("an index-excluded field must still be stored and readable per-record")
}
})
}
func TestIndexExcludePrefixes_EnvOverride(t *testing.T) {
if got := indexExcludePrefixes(); len(got) != 1 || got[0] != "categories/newznab/" {
t.Fatalf("unset env must yield the default, got %v", got)
}
t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "foo/, bar/ ,")
got := indexExcludePrefixes()
if len(got) != 2 || got[0] != "foo/" || got[1] != "bar/" {
t.Fatalf("override must trim and drop blanks, got %v", got)
}
// An explicitly empty value is the escape hatch: index everything.
t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "")
if got := indexExcludePrefixes(); len(got) != 0 {
t.Fatalf("an empty override must disable exclusion, got %v", got)
}
}
func TestHasAnyPrefix(t *testing.T) {
prefixes := []string{"categories/newznab/", "tmp/"}
cases := map[string]bool{
"categories/newznab/5070": true,
"tmp/scratch": true,
"categories/XXX": false,
"categories/newznab": false, // the family root is not the mirror
"title": false,
}
for field, want := range cases {
if got := hasAnyPrefix(field, prefixes); got != want {
t.Errorf("hasAnyPrefix(%q) = %v, want %v", field, got, want)
}
}
if hasAnyPrefix("anything", nil) {
t.Error("an empty prefix list must exclude nothing")
}
}
internal/storage/search_test.go
+327
View File
@@ -0,0 +1,327 @@
package storage
import (
"context"
"fmt"
"sort"
"strings"
"testing"
"github.com/metazla/meta-core/internal/kv"
)
// These tests pin the one property storage.Search must have: its SQL
// candidates are a SUPERSET of the records the exact matcher accepts. A
// candidate miss is a silently dropped search result, so each case below is a
// shape where SQLite and Go disagree about text (case folding, csv values,
// FTS syntax, trigram length) and the candidate query has to cover for it.
//
// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
// every candidate path; the API tests own the matcher itself.
func seed(t *testing.T, c *Client, records map[string]map[string]string) {
t.Helper()
for id, fields := range records {
if err := c.SetMetadataFlat(id, fields); err != nil {
t.Fatalf("seed %s: %v", id, err)
}
}
}
// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
for _, v := range values {
if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" {
return true
}
}
prefix := strings.ToLower(prop) + "/"
for k, val := range fields {
if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) {
for _, v := range values {
if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" {
return true
}
}
}
}
fv, ok := fields[prop]
if !ok {
return false
}
for _, piece := range strings.Split(fv, ",") {
piece = strings.TrimSpace(piece)
for _, v := range values {
if exact && strings.EqualFold(piece, v) {
return true
}
if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
return true
}
}
}
return false
}
func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string {
t.Helper()
hits, err := c.Search(query, filters, 100, match)
if err != nil {
t.Fatalf("search: %v", err)
}
ids := make([]string, 0, len(hits))
for _, h := range hits {
ids = append(ids, h.HashID)
}
return ids
}
func textMatch(query string) func(string, map[string]string) bool {
tokens := strings.Fields(strings.ToLower(query))
return func(id string, fields map[string]string) bool {
hay := Haystack(fields)
for _, tok := range tokens {
if !strings.Contains(hay, tok) {
return strings.Contains(strings.ToLower(id), strings.ToLower(query))
}
}
return true
}
}
func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool {
return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) }
}
func equalIDs(t *testing.T, got []string, want ...string) {
t.Helper()
sort.Strings(want)
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("got %v, want %v", got, want)
}
}
func TestSearchFilterCandidates(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"},
"R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"},
"R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"},
"R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"},
})
cases := []struct {
name string
prop string
exact bool
values []string
want []string
}{
{"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}},
{"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}},
{"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}},
{"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}},
{"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}},
{"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}},
{"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}},
{"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}},
{"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...))
equalIDs(t, got, tc.want...)
})
}
}
func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"R1": {"genres/Action": "true", "fileType": "video"},
"R2": {"genres/Action": "true", "fileType": "audio"},
})
match := func(_ string, f map[string]string) bool {
return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video")
}
got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match)
equalIDs(t, got, "R1")
}
func TestSearchFreeTextCandidates(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{
"01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"},
"01KBBB": {"title": "Dune: Part Two"},
"01KCCC": {"title": "進撃の巨人"},
"01KDDD": {"title": "3%", "titles/eng/3 percent": "true"},
"01KEEE": {"title": "AND -x a:b * \"quoted\""},
"01KFFF": {"title": "Unrelated"},
})
cases := []struct {
query string
want []string
}{
{"naruto", []string{"01KAAA"}},
{"NARUTO s01e02", []string{"01KAAA"}},
{"dune:", []string{"01KBBB"}},
{"巨人", []string{"01KCCC"}}, // 2 runes: below trigram length
{"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set
{"3%", []string{"01KDDD"}}, // LIKE metacharacter in a short token
{"AND", []string{"01KEEE"}}, // FTS operator as a literal
{"-x", []string{"01KEEE"}},
{"a:b", []string{"01KEEE"}},
{"*", []string{"01KEEE"}},
{`"quoted"`, []string{"01KEEE"}},
{"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring
}
for _, tc := range cases {
t.Run(tc.query, func(t *testing.T) {
got := search(t, c, tc.query, nil, textMatch(tc.query))
equalIDs(t, got, tc.want...)
})
}
}
// A rid token and a title token must not match together: the rid is in the
// FTS row only as a candidate path for the "whole query is a hashID substring"
// rule, and verification checks the two separately.
func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}})
if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 {
t.Fatalf("rid + title token matched: %v", got)
}
}
func TestSearchFollowsWritesAndDeletes(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
if err := c.SetProperty("R1", "title", "After"); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
equalIDs(t, search(t, c, "before", nil, textMatch("before")))
if _, err := c.DeleteMetadata("R1"); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "after", nil, textMatch("after")))
equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
}
func TestSearchPagesPastRejectedCandidates(t *testing.T) {
c := newTestClient(t)
records := map[string]map[string]string{}
for i := 0; i < 450; i++ {
id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3)
records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"}
}
records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"}
seed(t, c, records)
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}},
func(_ string, f map[string]string) bool { return f["keep"] == "yes" })
equalIDs(t, got, "RZZZZ")
}
// The filter candidates only scale if SQLite actually uses the partial
// expression index — which it does only when the query repeats the index's
// expression and WHERE clause as literals.
func TestSearchFilterUsesFieldIndex(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}})
q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}})
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") {
t.Fatalf("filter candidates do not use kv_field_v: %s", joined)
}
}
func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) {
if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" {
t.Fatalf("got %q %q %v", h, f, ok)
}
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}})
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video"))
equalIDs(t, got, "bafkreiabcdefghijklmnop")
}
func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) {
c := newTestClient(t)
seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}})
if err := c.update(func(tx *kv.Tx) error {
if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
return err
}
_, err := tx.DeletePrefix(ftsNS, 0)
return err
}); err != nil {
t.Fatal(err)
}
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")))
n, err := c.RebuildSearchIfEmpty()
if err != nil || n != 2 {
t.Fatalf("rebuilt %d (err %v), want 2", n, err)
}
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1")
if n, _ := c.RebuildSearchIfEmpty(); n != 0 {
t.Fatalf("second rebuild ran again (%d)", n)
}
}
// Text plus a broad filter: the selective part drives and the filter becomes a
// per-candidate probe over the record's own key range — never a full
// materialisation of the filter's result set.
func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
c := newTestClient(t)
recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}}
for i := 1; i < 300; i++ {
recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"}
}
seed(t, c, recs)
filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 {
t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes))
}
q, pargs := probeSQL(probes)
args := append([]any{`["R0000"]`}, pargs...)
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
if err != nil {
t.Fatal(err)
}
defer rows.Close()
var plan []string
for rows.Next() {
var id, parent, notused int
var detail string
if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
t.Fatal(err)
}
plan = append(plan, detail)
}
joined := strings.Join(plan, " | ")
if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
t.Fatalf("filter probe is not a primary-key range read: %s", joined)
}
equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool {
return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
}), "R0000")
}
internal/storage/stats.go
+28 -35
View File
@@ -1,52 +1,45 @@
package storage
import (
"context"
"fmt"
"time"
)
// CountRoots returns the number of metadata roots in the file index.
//
// SCARD, not SMEMBERS: GetAllHashIDs ships every member over the wire (~3MB
// and rising on an 88k-record box) just so the caller can take len(). A
// dashboard tile that polls needs the number, never the list.
// CountRoots returns the number of metadata roots in the root index — one
// range count over idx:roots/, never the list itself.
func (c *Client) CountRoots() (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.SCard(ctx, c.buildIndexKey()).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("scard failed: %w", err)
return 0, err
}
return n, nil
ctx, cancel := opCtx()
defer cancel()
return s.Count(ctx, rootsNS)
}
// CountKeys returns DBSIZE — every key in the Redis database, not just the
// ones meta-core owns. It is the denominator for "how much of Redis is
// records": ~3.1M keys against ~89k roots means the flat per-field keys
// (file:{root}/{field}) dominate, which is expected.
// keyCountTTL bounds how stale CountKeys may be. Counting every row is a full
// table scan (~a second at millions of keys), and the only reader is a polled
// dashboard tile — it needs a number, not a fresh one every few seconds.
const keyCountTTL = 60 * time.Second
// CountKeys returns the number of rows in the database — every key, not just
// records. It is the denominator for "how much of the store is records": the
// flat per-field keys (file:{root}/{field}) dominate, which is expected.
func (c *Client) CountKeys() (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
c.keysMu.Lock()
defer c.keysMu.Unlock()
if !c.keysAt.IsZero() && time.Since(c.keysAt) < keyCountTTL {
return c.keysCount, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.DBSize(ctx).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("dbsize failed: %w", err)
return 0, err
}
ctx, cancel := opCtx()
defer cancel()
n, err := s.CountAll(ctx)
if err != nil {
return 0, err
}
c.keysCount, c.keysAt = n, time.Now()
return n, nil
}
internal/storage/udl.go
+238 -277
View File
@@ -12,28 +12,31 @@ package storage
// strictly exceeds the stored one (a per-cell high-water-mark, replacing the
// op-log merge that used to live in meta-watch's store.rs).
//
// Redis model (all under the literal "udl:" prefix, mirroring the "file:" one):
// Key model (all under the literal "udl:" prefix, mirroring the "file:" one):
//
// udl:rec:<uid>/<cid>/<key> HASH { record, version, ts, value? }
// udl:idx:user:<uid>:key:<key> SET of <cid> -> active_for_user_key (My List, Continue Watching)
// udl:idx:user:<uid>:cid:<cid> SET of <key> -> list_for_user_cid
// udl:idx:user:<uid>:cids SET of <cid> -> whole-profile export (recovery)
// udl:idx:cid:<cid>:key:<key> SET of <uid> -> aggregate
// udl:idx:cid:<cid>:users SET of <uid> -> uids_for_cid
// udl:rec:<uid>/<cid>/<key> JSON { record, version, ts, value? }
// udl:idx:user:<uid>:key:<key> set of <cid> -> active_for_user_key (My List, Continue Watching)
// udl:idx:user:<uid>:cid:<cid> set of <key> -> list_for_user_cid
// udl:idx:user:<uid>:cids set of <cid> -> whole-profile export (recovery)
// udl:idx:cid:<cid>:key:<key> set of <uid> -> aggregate
// udl:idx:cid:<cid>:users set of <uid> -> uids_for_cid
//
// The `value` hash field is the plaintext JSON scalar for PUBLIC keys only
// A "set" is a key-set: one key `<set>\x1f<member>` = "" per member, read as
// the key range under `<set>\x1f`. The separator is a control character
// because UDL keys are not validated — with `/` the members of
// `…:key:a/b` would show up as members of `…:key:a`.
//
// The `value` field is the plaintext JSON scalar for PUBLIC keys only
// (like/rating), used for aggregation; it is absent for private-tier keys
// (whose real value rides encrypted inside `record`) and for tombstones.
import (
"context"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
"github.com/metazla/meta-core/internal/kv"
)
// UDLCell is a decoded (uid, cid, key) record cell.
@@ -67,7 +70,7 @@ func udlRecKey(uid, cid, key string) string {
func udlIdxUserKey(uid, key string) string { return fmt.Sprintf("udl:idx:user:%s:key:%s", uid, key) }
func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s:cid:%s", uid, cid) }
func udlIdxCidKey(cid, key string) string { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
// index that makes "export this whole profile" a set read instead of a
@@ -81,161 +84,196 @@ func udlIdxCidUsers(cid string) string { return fmt.Sprintf("udl:idx:cid:%s
// already made `/api/udl/users/stats` admin-only.
func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
// udlUpsertScript performs the version-gated upsert + index maintenance
// atomically. Accepts (returns 1) only when the incoming version strictly
// exceeds the stored one; otherwise returns 0 (stale).
//
// KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
// 6=idx:user:cids
// ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
// 6=cid 7=key 8=uid 9=tombstone("1"/"0")
//
// A tombstone still writes the cell — that is the CRDT state, and dropping it
// would let an older write resurrect the value — but it SREMs the cid from the
// user+key index. That index is what `active_for_user_key` returns, so leaving
// deleted cids in it means every My List / Continue Watching read carries every
// title ever un-liked or finished, forever. The other four indexes are left
// alone: they answer "who has an opinion about this cid", where a tombstone is
// still an answer.
//
// ⚠ `idx:user:cids` (KEYS[6]) in particular must NEVER be pruned on a
// tombstone, even though it looks symmetric with KEYS[2]. It is the export
// index: a cid dropped from it is a cid whose tombstone never travels, and a
// tombstone that does not travel is exactly how an un-liked title comes back
// from the dead on the other box — the peer still holds the older `like` at a
// lower version, and with no tombstone to beat it, nothing ever supersedes it.
var udlUpsertScript = redis.NewScript(`
local stored = redis.call('HGET', KEYS[1], 'version')
if stored and tonumber(stored) >= tonumber(ARGV[1]) then
return 0
end
redis.call('HSET', KEYS[1], 'version', ARGV[1], 'ts', ARGV[2], 'record', ARGV[3])
if ARGV[5] == '1' then
redis.call('HSET', KEYS[1], 'value', ARGV[4])
else
redis.call('HDEL', KEYS[1], 'value')
end
if ARGV[9] == '1' then
redis.call('SREM', KEYS[2], ARGV[6])
else
redis.call('SADD', KEYS[2], ARGV[6])
end
redis.call('SADD', KEYS[3], ARGV[7])
redis.call('SADD', KEYS[4], ARGV[8])
redis.call('SADD', KEYS[5], ARGV[8])
redis.call('SADD', KEYS[6], ARGV[6])
return 1
`)
// udlSetSep separates a set key from a member (see the key model above).
const udlSetSep = "\x1f"
// udlCellJSON is a cell's stored form. Value is a pointer so an absent value
// (private-tier key, tombstone) stays distinguishable from an empty one.
type udlCellJSON struct {
Record string `json:"record"`
Version int64 `json:"version"`
Ts int64 `json:"ts"`
Value *string `json:"value,omitempty"`
}
func decodeUDLCell(raw string) (UDLCell, error) {
var j udlCellJSON
if err := json.Unmarshal([]byte(raw), &j); err != nil {
return UDLCell{}, err
}
cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts}
if j.Value != nil {
cell.Value, cell.HasValue = *j.Value, true
}
return cell, nil
}
// udlMembers lists a key-set's members, sorted.
func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) {
prefix := set + udlSetSep
rows, err := r.Range(ctx, prefix, "", 0)
if err != nil {
return nil, err
}
out := make([]string, 0, len(rows))
for _, row := range rows {
out = append(out, row.K[len(prefix):])
}
return out, nil
}
func udlAdd(tx *kv.Tx, set, member string) error {
_, err := tx.PutIfAbsent(set+udlSetSep+member, "")
return err
}
func udlRem(tx *kv.Tx, set, member string) error {
_, err := tx.Delete(set + udlSetSep + member)
return err
}
// readUDLCell reads one cell. exists is false when it was never written.
func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) {
raw, ok, err := r.Get(ctx, recKey)
if err != nil || !ok {
return UDLCell{}, false, err
}
cell, err := decodeUDLCell(raw)
if err != nil {
return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err)
}
return cell, true, nil
}
// readUDLCells reads many cells in one multi-key read; absent cells are
// omitted from the map.
func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) {
raws, err := r.MGet(ctx, recKeys)
if err != nil {
return nil, err
}
out := make(map[string]UDLCell, len(raws))
for k, raw := range raws {
cell, err := decodeUDLCell(raw)
if err != nil {
return nil, fmt.Errorf("decode %s: %w", k, err)
}
out[k] = cell
}
return out, nil
}
// UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
// one. publicValue/hasValue carry the plaintext JSON scalar for public keys
// (used by aggregation); pass hasValue=false for private-tier keys and
// tombstones. Set tombstone=true when the record deletes the cell, which also
// prunes the cid from the user+key index. Returns accepted=false when the write
// is stale.
// one, and maintains the indexes — one transaction, so the gate and the write
// cannot interleave with a concurrent upsert of the same cell.
// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by
// aggregation); pass hasValue=false for private-tier keys and tombstones. Set
// tombstone=true when the record deletes the cell.
//
// A tombstone still writes the cell — that is the CRDT state, and dropping it
// would let an older write resurrect the value — but it removes the cid from
// the user+key index. That index is what `active_for_user_key` returns, so
// leaving deleted cids in it means every My List / Continue Watching read
// carries every title ever un-liked or finished, forever. The other four
// indexes are left alone: they answer "who has an opinion about this cid",
// where a tombstone is still an answer.
//
// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even
// though it looks symmetric with the user+key index. It is the export index: a
// cid dropped from it is a cid whose tombstone never travels, and a tombstone
// that does not travel is exactly how an un-liked title comes back from the
// dead on the other box — the peer still holds the older `like` at a lower
// version, and with no tombstone to beat it, nothing ever supersedes it.
//
// Returns accepted=false when the write is stale.
func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
keys := []string{
c.buildKey(udlRecKey(uid, cid, key)),
c.buildKey(udlIdxUserKey(uid, key)),
c.buildKey(udlIdxUserCid(uid, cid)),
c.buildKey(udlIdxCidKey(cid, key)),
c.buildKey(udlIdxCidUsers(cid)),
c.buildKey(udlIdxUserCids(uid)),
}
hv := "0"
if hasValue {
hv = "1"
}
tomb := "0"
if tombstone {
tomb = "1"
}
res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
accepted := false
err := c.update(func(tx *kv.Tx) error {
recKey := udlRecKey(uid, cid, key)
stored, ok, err := readUDLCell(tx.Context(), tx, recKey)
if err != nil {
return err
}
if ok && stored.Version >= version {
return nil
}
cell := udlCellJSON{Record: record, Version: version, Ts: ts}
if hasValue {
v := publicValue
cell.Value = &v
}
body, err := json.Marshal(cell)
if err != nil {
return err
}
if err := tx.Put(recKey, string(body)); err != nil {
return err
}
if tombstone {
err = udlRem(tx, udlIdxUserKey(uid, key), cid)
} else {
err = udlAdd(tx, udlIdxUserKey(uid, key), cid)
}
if err != nil {
return err
}
for _, add := range [][2]string{
{udlIdxUserCid(uid, cid), key},
{udlIdxCidKey(cid, key), uid},
{udlIdxCidUsers(cid), uid},
{udlIdxUserCids(uid), cid},
} {
if err := udlAdd(tx, add[0], add[1]); err != nil {
return err
}
}
accepted = true
return nil
})
if err != nil {
return false, fmt.Errorf("udl upsert failed: %w", err)
}
n, _ := res.(int64)
return n == 1, nil
return accepted, nil
}
// UDLGetCell returns the record cell for (uid, cid, key). exists is false when
// no record has ever been written for the cell.
func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return UDLCell{}, false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.udlReadCell(ctx, udlRecKey(uid, cid, key))
}
// udlReadCell HMGETs a cell hash. Caller holds the lock. exists is false when
// the version field is absent (i.e. the hash does not exist).
func (c *Client) udlReadCell(ctx context.Context, recKey string) (UDLCell, bool, error) {
vals, err := c.client.HMGet(ctx, c.buildKey(recKey), "record", "version", "ts", "value").Result()
s, err := c.kv()
if err != nil {
return UDLCell{}, false, fmt.Errorf("hmget failed: %w", err)
return UDLCell{}, false, err
}
// vals: [record, version, ts, value] — nil for absent fields.
if len(vals) < 4 || vals[1] == nil {
return UDLCell{}, false, nil
}
cell := UDLCell{}
if s, ok := vals[0].(string); ok {
cell.Record = s
}
if s, ok := vals[1].(string); ok {
cell.Version, _ = strconv.ParseInt(s, 10, 64)
}
if s, ok := vals[2].(string); ok {
cell.Ts, _ = strconv.ParseInt(s, 10, 64)
}
if s, ok := vals[3].(string); ok {
cell.Value = s
cell.HasValue = true
}
return cell, true, nil
ctx, cancel := opCtx()
defer cancel()
return readUDLCell(ctx, s, udlRecKey(uid, cid, key))
}
// UDLListUserKey returns every cell for (uid, key) across all cids — the
// projection behind active_for_user_key (My List, Continue Watching). Includes
// tombstones; the caller decodes each record and filters.
func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserKey(uid, key))).Result()
cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(cids))
for i, cid := range cids {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]UDLUserKeyEntry, 0, len(cids))
for _, cid := range cids {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
if err != nil {
return nil, err
}
for i, cid := range cids {
cell, ok := cells[recKeys[i]]
if !ok {
continue
}
@@ -247,26 +285,28 @@ func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
// UDLListUserCid returns every cell for (uid, cid) across all keys — the
// projection behind list_for_user_cid.
func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(keys))
for i, key := range keys {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]UDLUserCidEntry, 0, len(keys))
for _, key := range keys {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
if err != nil {
return nil, err
}
for i, key := range keys {
cell, ok := cells[recKeys[i]]
if !ok {
continue
}
@@ -286,33 +326,28 @@ type UDLUserAllEntry struct {
// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
//
// Ordered by cid (sorted, so paging is stable across calls even though the
// underlying index is a Redis SET), starting strictly after `afterCid`. A cid's
// keys are never split across a page: the page ends on a cid boundary once
// `limit` rows have been emitted, and `nextCid` is the last cid included — pass
// it back as `afterCid` to continue. `nextCid` is empty when the export is
// complete.
// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never
// split across a page: the page ends on a cid boundary once `limit` rows have
// been emitted, and `nextCid` is the last cid included — pass it back as
// `afterCid` to continue. `nextCid` is empty when the export is complete.
//
// Tombstones are included. They are the CRDT state that stops a deleted value
// resurrecting on the box that receives this export, so omitting them would
// make recovery actively wrong rather than merely incomplete.
func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, "", fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, "", err
}
if limit <= 0 {
limit = 1000
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
ctx, cancel := opCtx()
defer cancel()
cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
cids, err := udlMembers(ctx, s, udlIdxUserCids(uid))
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
return nil, "", err
}
sort.Strings(cids)
@@ -326,13 +361,12 @@ func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEnt
nextCid = out[len(out)-1].Cid
break
}
keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
if err != nil {
return nil, "", fmt.Errorf("smembers failed: %w", err)
return nil, "", err
}
sort.Strings(keys)
for _, key := range keys {
cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key))
if err != nil {
return nil, "", err
}
@@ -349,133 +383,60 @@ func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEnt
// UDLCountUserCids is the number of distinct cids a uid holds records for.
//
// O(1) (a Redis SCARD), which is what makes it usable as a per-push quota
// One index range count, which is what makes it usable as a per-push quota
// check on the gateway mirror — a quota that cost a keyspace walk would be a
// denial-of-service in its own right. Counts cids rather than cells because
// that is what the index holds; a profile is bounded by the titles it has
// touched, not by how many keys it has per title.
func (c *Client) UDLCountUserCids(uid string) (int64, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.SCard(ctx, c.buildKey(udlIdxUserCids(uid))).Result()
s, err := c.kv()
if err != nil {
return 0, fmt.Errorf("scard failed: %w", err)
return 0, err
}
return n, nil
}
// UDLBackfillUserCids rebuilds the `idx:user:<uid>:cids` sets for data written
// before that index existed, and reports how many cids it added.
//
// One-shot: the upsert script maintains the set from now on. It is a SCAN, but
// a targeted one (`udl:idx:user:*:cid:*`, which is one key per (uid,cid), not
// per record) run once at boot rather than per read — the distinction that
// makes the difference between a migration and the pathology this index exists
// to avoid.
func (c *Client) UDLBackfillUserCids() (int, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
ctx, cancel := opCtx()
defer cancel()
pattern := c.buildKey("udl:idx:user:*:cid:*")
prefix := c.buildKey("udl:idx:user:")
added := 0
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 500).Result()
if err != nil {
return added, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
// <prefix><uid>:cid:<cid> — split on the FIRST separator, not the
// last. A uid is multibase base58btc, whose alphabet contains no
// colon at all, so the first ":cid:" always ends the uid. A cid
// that happens to contain ":cid:" in its own name would make the
// last occurrence split in the wrong place.
rest := strings.TrimPrefix(k, prefix)
i := strings.Index(rest, ":cid:")
if i < 0 {
continue
}
uid, cid := rest[:i], rest[i+len(":cid:"):]
if uid == "" || cid == "" {
continue
}
n, err := c.client.SAdd(ctx, c.buildKey(udlIdxUserCids(uid)), cid).Result()
if err != nil {
return added, fmt.Errorf("sadd failed: %w", err)
}
added += int(n)
}
cursor = next
if cursor == 0 {
break
}
}
return added, nil
return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep)
}
// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
func (c *Client) UDLCidUsers(cid string) ([]string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidUsers(cid))).Result()
s, err := c.kv()
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
return uids, nil
ctx, cancel := opCtx()
defer cancel()
return udlMembers(ctx, s, udlIdxCidUsers(cid))
}
// UDLPublicValues returns the live plaintext public JSON scalar values for
// (cid, key) across all uids — the input to aggregation. Cells without a value
// field (private-tier or tombstoned) are skipped.
// (private-tier or tombstoned) are skipped.
func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := opCtx()
defer cancel()
uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidKey(cid, key))).Result()
uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key))
if err != nil {
return nil, fmt.Errorf("smembers failed: %w", err)
return nil, err
}
recKeys := make([]string, len(uids))
for i, uid := range uids {
recKeys[i] = udlRecKey(uid, cid, key)
}
cells, err := readUDLCells(ctx, s, recKeys)
if err != nil {
return nil, err
}
out := make([]string, 0, len(uids))
for _, uid := range uids {
v, err := c.client.HGet(ctx, c.buildKey(udlRecKey(uid, cid, key)), "value").Result()
if err == redis.Nil {
continue // private-tier or tombstoned
for _, k := range recKeys {
if cell, ok := cells[k]; ok && cell.HasValue {
out = append(out, cell.Value)
}
if err != nil {
return nil, fmt.Errorf("hget failed: %w", err)
}
out = append(out, v)
}
return out, nil
}
internal/storage/udl_admin.go
+76 -144
View File
@@ -3,22 +3,16 @@ package storage
// Account-scoped UDL administration: what a uid holds, and removing all of it.
//
// Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
// cell — because there is no per-user cid index to consult. The Redis model
// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
// "what is in My List", not "everything this account ever wrote". Adding a
// third index would put a write on every UDL upsert to serve two admin calls,
// so these SCAN instead and accept being O(keyspace).
//
// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
// one: the accounts page needs a number for every account, and MATCH filters
// server-side without indexing, so N per-uid calls would walk the keyspace N
// times. One walk buckets them all.
// cell. Keys are ordered, so one account's cells are the key range under
// `udl:rec:<uid>/`, and every account's are the range under `udl:rec:`.
// UDLAllUserStats stays a bulk call (one walk buckets every account) because
// the accounts page needs a number for each.
import (
"context"
"fmt"
"strings"
"time"
"github.com/metazla/meta-core/internal/kv"
)
// UDLUserStats counts what one account holds. Records is the number of cells,
@@ -30,11 +24,8 @@ type UDLUserStats struct {
Keys int `json:"keys"`
}
// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
const udlScanBatch = 1000
// safeUIDForPattern rejects anything that would change the meaning of a SCAN
// glob or reach across the key structure. meta-core's uids are alphanumeric
// safeUIDForPattern rejects anything that would reach across the key
// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric
// (identity.validUID), so this only ever fires on a hostile or malformed one —
// but it fires here, in the layer that builds the pattern, rather than relying
// on every caller having validated first.
@@ -62,118 +53,76 @@ func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
return parts[0], parts[1], parts[2], true
}
// scanKeys walks the keyspace for pattern and returns every match. Caller holds
// the lock.
//
// SCAN can return the same key more than once across cursor iterations (it
// guarantees no misses, not no duplicates), so callers that count must
// de-duplicate — both callers below collect into maps or sets for that reason.
func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
var (
out []string
cursor uint64
)
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
if err != nil {
return nil, fmt.Errorf("scan %q: %w", pattern, err)
}
out = append(out, keys...)
cursor = next
if cursor == 0 {
return out, nil
}
}
}
// UDLUserCells returns every (cid, key) pair this account holds a cell for.
// De-duplicated, since SCAN may repeat a key.
// UDLUserCells returns every (cid, key) pair this account holds a cell for,
// keyed by the "cid/key" tail with the cid as value (the purge needs them
// grouped).
func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
// the cid, which the purge needs grouped.
if err := safeUIDForPattern(uid); err != nil {
return nil, err
}
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+uid+"/*")
s, err := c.kv()
if err != nil {
return nil, err
}
out := make(map[string]string, len(keys))
for _, full := range keys {
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
// The glob can only match this uid, but a uid that is a prefix of
// another would still be caught by the '/' in the pattern; check anyway
// so a malformed key can never be attributed to the wrong account.
if !ok || gotUID != uid {
continue
ctx, cancel := opCtx()
defer cancel()
out := make(map[string]string)
err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error {
for _, row := range rows {
gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
if !ok || gotUID != uid {
continue
}
out[cid+"/"+key] = cid
}
out[cid+"/"+key] = cid
}
return out, nil
return nil
})
return out, err
}
// UDLAllUserStats counts every account's cells in one keyspace walk.
// UDLAllUserStats counts every account's cells in one walk.
//
// Returns only accounts that actually hold records; a uid absent from the map
// holds none. Callers render a zero for those rather than treating it as an
// error — an account created a minute ago legitimately has nothing.
func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
prefix := c.buildKey("udl:rec:")
keys, err := c.scanKeys(ctx, prefix+"*")
s, err := c.kv()
if err != nil {
return nil, err
}
ctx, cancel := opCtx()
defer cancel()
// Sets rather than counters: SCAN may hand back the same key twice, which
// would otherwise inflate every number on the page.
type acc struct {
cells map[string]struct{}
cells int
cids map[string]struct{}
keys map[string]struct{}
}
byUID := make(map[string]*acc)
for _, full := range keys {
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
if !ok {
continue
}
a := byUID[uid]
if a == nil {
a = &acc{
cells: make(map[string]struct{}),
cids: make(map[string]struct{}),
keys: make(map[string]struct{}),
err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error {
for _, row := range rows {
uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
if !ok {
continue
}
byUID[uid] = a
a := byUID[uid]
if a == nil {
a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})}
byUID[uid] = a
}
a.cells++
a.cids[cid] = struct{}{}
a.keys[key] = struct{}{}
}
a.cells[cid+"/"+key] = struct{}{}
a.cids[cid] = struct{}{}
a.keys[key] = struct{}{}
return nil
})
if err != nil {
return nil, err
}
out := make(map[string]UDLUserStats, len(byUID))
for uid, a := range byUID {
out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)}
}
return out, nil
}
@@ -185,63 +134,46 @@ func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
// The shared indexes are the part that is easy to miss and expensive to leave.
// `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
// not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
// uid listed as an opinion-holder on every title it ever touched. Aggregation
// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
// rating average stays right, but uids_for_cid keeps naming a deleted account
// forever and every aggregate read pays for the lookup.
// uid listed as an opinion-holder on every title it ever touched.
//
// Idempotent: purging an account with nothing stored is a no-op returning
// zeroes, so a retry after a partial failure is safe.
// zeroes, so a retry is safe.
func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
cells, err := c.UDLUserCells(uid)
if err != nil {
return UDLUserStats{}, err
}
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return UDLUserStats{}, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
stats := UDLUserStats{Records: len(cells)}
distinctCids := make(map[string]struct{})
distinctKeys := make(map[string]struct{})
pipe := c.client.Pipeline()
for cidKey, cid := range cells {
key := strings.TrimPrefix(cidKey, cid+"/")
distinctCids[cid] = struct{}{}
distinctKeys[key] = struct{}{}
pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
}
for cid := range distinctCids {
pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
err = c.update(func(tx *kv.Tx) error {
for cidKey, cid := range cells {
key := strings.TrimPrefix(cidKey, cid+"/")
distinctCids[cid] = struct{}{}
distinctKeys[key] = struct{}{}
if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil {
return err
}
if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil {
return err
}
}
for cid := range distinctCids {
if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil {
return err
}
}
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one
// key range, dropped whole — which also catches index entries whose
// cell is already gone (the state a half-finished purge leaves).
_, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0)
return err
})
if err != nil {
return UDLUserStats{}, fmt.Errorf("purge: %w", err)
}
stats.Cids = len(distinctCids)
stats.Keys = len(distinctKeys)
// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are enumerable
// directly, and are dropped whole rather than member-by-member. The glob is
// what keeps this correct as indexes are added — `:cids` needed no change
// here when it landed. Scanning them separately also catches
// index entries whose cell is already gone — exactly the state a previously
// half-finished purge leaves behind.
idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
if err != nil {
return UDLUserStats{}, err
}
for _, k := range idxKeys {
pipe.Del(ctx, k)
}
if _, err := pipe.Exec(ctx); err != nil {
return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
}
return stats, nil
}
internal/storage/udl_admin_test.go
+8 -28
View File
@@ -20,7 +20,7 @@ func seedCell(t *testing.T, c *Client, uid, cid, key string) {
// forever — and the rows are unreachable through any API, so nothing will ever
// clean them up.
func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -63,7 +63,7 @@ func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
// both have an opinion about — where the two share an index set and a careless
// DEL of the whole key would take the survivor's membership with it.
func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "shared", "rating")
seedCell(t, c, "zBob", "shared", "rating")
@@ -92,7 +92,7 @@ func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
// The key layout separates them with '/' and ':', so this passes only as long
// as the scan patterns keep those separators.
func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAliceExtra", "cidOne", "like")
@@ -114,7 +114,7 @@ func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
// index instead of the cells would leave every retracted rating behind — the
// rows most likely to have accumulated over an account's life.
func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
@@ -140,7 +140,7 @@ func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
// Retry-safe: the delete handler purges before removing the key, so a caller
// that retries after a partial failure runs this a second time.
func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
if _, err := c.UDLPurgeUser("zAlice"); err != nil {
@@ -156,7 +156,7 @@ func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
}
func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
// "*" would otherwise match every account in the keyspace — a wildcard
@@ -172,7 +172,7 @@ func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
}
func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -198,7 +198,7 @@ func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
// The stats and the purge must agree: the number shown in the delete
// confirmation is the number of rows the delete then removes.
func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
seedCell(t, c, "zAlice", "cidOne", "like")
seedCell(t, c, "zAlice", "cidTwo", "rating")
@@ -217,26 +217,6 @@ func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
}
}
// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or
// every account on a prefixed node would be reported under a mangled name and
// the purge would find nothing.
func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) {
c, _ := newTestClient(t, "tenantA:")
seedCell(t, c, "zAlice", "cidOne", "like")
stats, err := c.UDLAllUserStats()
if err != nil {
t.Fatalf("stats: %v", err)
}
if got := stats["zAlice"]; got.Records != 1 {
t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats)
}
if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 {
t.Fatalf("purge under a prefix: %+v err=%v", s, err)
}
}
func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
// A namespaced key arriving later must not be truncated: the purge SREMs
// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong
internal/storage/udl_export_test.go
+4 -78
View File
@@ -1,7 +1,6 @@
package storage
import (
"context"
"testing"
)
@@ -11,7 +10,7 @@ import (
// that one caller.
func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const uid = "zAlice"
writes := []struct{ cid, key, rec string }{
@@ -52,7 +51,7 @@ func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
// indexed exports as empty, and recovery reports success having restored
// nothing.
func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
@@ -74,7 +73,7 @@ func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
// saw the `like`. This is the one place the cids index deliberately diverges
// from the user+key index, which does prune.
func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const (
uid = "zAlice"
@@ -111,7 +110,7 @@ func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
}
func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const uid = "zAlice"
cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
@@ -159,76 +158,3 @@ func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
seen[k] = true
}
}
// Data written before the cids index existed must still export, or every
// profile that predates this feature silently recovers as empty.
func TestUDLBackfillUserCids_MakesLegacyDataExportable(t *testing.T) {
c, _ := newTestClient(t, "")
const (
uid = "zAlice"
cid = "bagcsaaaaone"
key = "like"
)
if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
// Simulate the pre-index state: the cell and the other indexes exist, the
// cids set does not.
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if entries, _, err := c.UDLAllForUser(uid, "", 0); err != nil || len(entries) != 0 {
t.Fatalf("precondition: legacy data should export as empty, got %+v err=%v", entries, err)
}
added, err := c.UDLBackfillUserCids()
if err != nil {
t.Fatalf("backfill: %v", err)
}
if added != 1 {
t.Fatalf("want 1 cid backfilled, got %d", added)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export after backfill: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid || entries[0].Key != key {
t.Fatalf("backfilled data should export, got %+v", entries)
}
// Idempotent: a second run adds nothing.
if added, err := c.UDLBackfillUserCids(); err != nil || added != 0 {
t.Fatalf("second backfill should be a no-op, added=%d err=%v", added, err)
}
}
// The backfill splits `<uid>:cid:<cid>` on the FIRST separator, because a uid
// is multibase base58btc and its alphabet has no colon. Splitting on the last
// occurrence — the obvious-looking choice — mis-parses a cid that contains
// ":cid:" in its own name into uid "zAlice:cid:weird", cid "name".
func TestUDLBackfillUserCids_SplitsOnTheFirstSeparator(t *testing.T) {
c, _ := newTestClient(t, "")
const uid = "zAlice"
cid := "weird:cid:name"
if ok, err := c.UDLUpsertIfNewer(uid, cid, "like", 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("upsert: ok=%v err=%v", ok, err)
}
ctx := context.Background()
if err := c.client.Del(ctx, c.buildKey(udlIdxUserCids(uid))).Err(); err != nil {
t.Fatalf("del cids index: %v", err)
}
if _, err := c.UDLBackfillUserCids(); err != nil {
t.Fatalf("backfill: %v", err)
}
entries, _, err := c.UDLAllForUser(uid, "", 0)
if err != nil {
t.Fatalf("export: %v", err)
}
if len(entries) != 1 || entries[0].Cid != cid {
t.Fatalf("want the cid parsed whole, got %+v", entries)
}
}
internal/storage/udl_test.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
// forever. The reader filters them out again, so the cost is invisible until
// the account is old.
func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
c, _ := newTestClient(t, "")
c := newTestClient(t)
const (
uid = "zAlice"
internal/storage/uuid.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
// layout). Used as the root key for file metadata entries.
//
// Why this format over canonical 8-4-4-4-12 hex:
// - 26 chars vs 36 — shorter Redis keys, less prefix space waste
// - 26 chars vs 36 — shorter keys, less prefix space waste
// - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
// given time window, restoring some of the debug-ability lost vs the
// previous midhash256:abc rooting
internal/storage/uuid_test.go
+5 -5
View File
@@ -60,11 +60,11 @@ func TestCIDFromKeysetField(t *testing.T) {
{"cids/bafkmidhash", "bafkmidhash"},
{"cids/baejbeisha256", "baejbeisha256"},
{"cids/bafyipfs", "bafyipfs"},
{"title", ""}, // non-CID field
{"tmdb/title", ""}, // non-CID nested
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
{"midhash256", ""}, // legacy named field — no longer recognized
{"cids", ""}, // prefix without a member
{"title", ""}, // non-CID field
{"tmdb/title", ""}, // non-CID nested
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
{"midhash256", ""}, // legacy named field — no longer recognized
{"cids", ""}, // prefix without a member
}
for _, tc := range cases {
if got := cidFromKeysetField(tc.field); got != tc.want {
internal/storage/write_bench_test.go
+59
View File
@@ -0,0 +1,59 @@
package storage
import (
"fmt"
"path/filepath"
"sync"
"sync/atomic"
"testing"
)
// BenchmarkConcurrentRecordWrites is the ingest shape the single-writer model
// has to absorb: many goroutines (feeder flushes, gateway persists, plugin
// callbacks) each writing a ~40-field record. SQLite serialises them; this
// measures what that costs. Reported as records/s.
func BenchmarkConcurrentRecordWrites(b *testing.B) {
c := NewClient()
if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
b.Fatal(err)
}
defer c.Close()
record := func(i int64) map[string]string {
m := map[string]string{
"title": fmt.Sprintf("Some Show S01E%02d", i%40),
"fileName": fmt.Sprintf("some.show.s01e%02d.1080p.mkv", i%40),
"filePath": fmt.Sprintf("/files/watch/some.show.%d.mkv", i),
"fileType": "video",
"tmdbid": fmt.Sprintf("%d", 1000+i%500),
fmt.Sprintf("cids/bafkbench%020d", i): "true",
}
for f := 0; len(m) < 40; f++ {
m[fmt.Sprintf("categories/%d", f)] = "true"
}
return m
}
const writers = 16
var next int64
b.ResetTimer()
var wg sync.WaitGroup
for w := 0; w < writers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
i := atomic.AddInt64(&next, 1)
if i > int64(b.N) {
return
}
if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), record(i)); err != nil {
b.Error(err)
return
}
}
}()
}
wg.Wait()
b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
}
internal/watcher/dispatcher.go
+10 -10
View File
@@ -11,7 +11,7 @@ import (
)
const (
// EventsStream is the Redis Stream name for file events
// EventsStream is the stream name for file events
EventsStream = "file:events"
// EventsStreamMaxLen is the approximate cap on stream length. Bounded
@@ -20,7 +20,7 @@ const (
EventsStreamMaxLen = 100_000
)
// Dispatcher sends file events to Redis Stream
// Dispatcher sends file events to the file:events stream
type Dispatcher struct {
mu sync.RWMutex
storageClient *storage.Client
@@ -31,20 +31,20 @@ func NewDispatcher() *Dispatcher {
return &Dispatcher{}
}
// SetStorageClient sets the storage client for Redis Stream publishing
// SetStorageClient sets the storage client for file:events publishing
func (d *Dispatcher) SetStorageClient(client *storage.Client) {
d.mu.Lock()
defer d.mu.Unlock()
d.storageClient = client
log.Println("[Dispatcher] Storage client set for Redis Stream publishing")
log.Println("[Dispatcher] Storage client set for file:events publishing")
}
// Dispatch sends an event to Redis Stream
// Dispatch sends an event to the file:events stream
func (d *Dispatcher) Dispatch(event FileEvent) {
go d.dispatchToStream(event)
}
// dispatchToStream publishes event to Redis Stream for service consumption
// dispatchToStream publishes event to the file:events stream for service consumption
func (d *Dispatcher) dispatchToStream(event FileEvent) {
d.mu.RLock()
client := d.storageClient
@@ -114,7 +114,7 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
// Emit reset event as the first message in the fresh stream
event := FileEvent{
Type: EventTypeReset,
Path: "", // No specific path for reset
Path: "", // No specific path for reset
WatcherID: watcherID,
Timestamp: NowMS(),
}
@@ -125,7 +125,7 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
return nil
}
// GetRecentEvents reads recent events from the Redis stream, newest first.
// GetRecentEvents reads recent events from the stream, newest first.
// If sinceMS > 0, only events strictly newer than that timestamp are returned.
// limit caps the number of returned events (defaults to 100 when <= 0).
func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
@@ -160,7 +160,7 @@ func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, err
return events, nil
}
// parseStreamEntry converts a Redis stream entry back into a FileEvent.
// parseStreamEntry converts a stream entry back into a FileEvent.
// All fields are stored as strings by XADD, so numerics need parsing.
func parseStreamEntry(entry storage.StreamEntry) FileEvent {
event := FileEvent{}
@@ -192,7 +192,7 @@ func parseStreamEntry(entry storage.StreamEntry) FileEvent {
}
// Fall back to the stream ID's millisecond prefix if the timestamp
// field is missing or unparseable — every Redis stream ID starts with ms.
// field is missing or unparseable — every stream ID starts with ms.
if event.Timestamp == 0 && entry.ID != "" {
if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {
internal/watcher/handlers.go
+2 -2
View File
@@ -25,7 +25,7 @@ func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
// RegisterRoutes registers all watcher-related routes
func (h *Handlers) RegisterRoutes(r *mux.Router) {
// Event polling (for services that don't use Redis Streams)
// Event polling (for services that don't consume the SSE stream)
r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
// Deprecated scan endpoints - redirect to /api/watchers
@@ -61,7 +61,7 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
}
}
// Read events from the Redis `file:events` stream rather than the
// Read events from the `file:events` stream rather than the
// in-memory buffer. The buffer is fed by the unused debouncer path,
// so it is always empty in practice — the stream is the source of truth.
events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)
internal/watcher/types.go
+1 -1
View File
@@ -16,7 +16,7 @@ const (
// FileEvent represents a file system event
type FileEvent struct {
Type FileEventType `json:"type"`
Path string `json:"path"` // Relative to FILES_PATH
Path string `json:"path"` // Relative to FILES_PATH
Size int64 `json:"size,omitempty"`
Timestamp int64 `json:"timestamp"`
MidHash256 string `json:"midhash256,omitempty"` // midhash256 CID (SHA-256 of middle 1MB + file size)
internal/watcher/watcher.go
+6 -8
View File
@@ -11,7 +11,6 @@ import (
"github.com/metazla/meta-core/internal/storage"
)
// Watcher provides file scanning and event dispatching
type Watcher struct {
config *config.Config
@@ -26,8 +25,8 @@ type Watcher struct {
}
// SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
// tuple to Redis whenever it computes a midhash. Without this, only the
// in-memory state registry knows about the file — Redis would only learn
// tuple to the store whenever it computes a midhash. Without this, only the
// in-memory state registry knows about the file — the store would only learn
// about it via downstream consumers of the event stream.
func (w *Watcher) SetStorageClient(s *storage.Client) {
w.storage = s
@@ -35,7 +34,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
// writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
// Caller already has all four values from its existing os.Stat / FileInfo —
// no extra IO is performed here beyond the Redis writes themselves.
// no extra IO is performed here beyond the store writes themselves.
//
// Flow:
// - Look up the midhash CID in the reverse index (cid:<midhash> → uuid)
@@ -46,7 +45,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
// record the new path in the duplicates set and refresh size/mtime
//
// This is also the inverse-index that lets a fresh meta-core boot skip
// re-mid-hashing files whose (path, size, mtime) match what's in Redis,
// re-mid-hashing files whose (path, size, mtime) match what's stored,
// via HydrateStateFromStorage below.
func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
if w.storage == nil || midhash == "" {
@@ -270,7 +269,7 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
}
// WatcherRoot pairs a watcher's id with the absolute path it watches. The
// hydrator uses Root to decide which Redis-tracked file belongs to which
// hydrator uses Root to decide which stored file belongs to which
// in-memory WatcherState.
type WatcherRoot struct {
ID string
@@ -278,7 +277,7 @@ type WatcherRoot struct {
}
// HydrateStateFromStorage repopulates the per-watcher in-memory state from
// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
// the (filePath, sizeByte, mtimeNano) tuples in the store. After this runs, the
// next scan tick will see existing files as "known" and skip
// ComputeMidHash256 for any whose size+mtime match — turning a cold-start
// rescan from minutes to seconds on a populated library.
@@ -416,4 +415,3 @@ func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
func (w *Watcher) GetFilesPath() string {
return w.filesPath
}
internal/watchers/handlers.go
+3 -3
View File
@@ -227,9 +227,9 @@ func (h *Handlers) handleResetAll(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"message": "All watchers reset",
"totalFiles": totalCount,
"status": "ok",
"message": "All watchers reset",
"totalFiles": totalCount,
"republishedEvents": republishedCount,
})
}
internal/watchers/poller.go
+2 -2
View File
@@ -74,7 +74,7 @@ func (p *Poller) Start() error {
// consumer state without buying anything. The historical reason for
// wiping — "in-memory state is empty, all files re-emit as add" —
// stopped being true when HydrateStateFromStorage landed: state is
// hydrated from Redis before this runs, so a startup EmitReset would
// hydrated from the store before this runs, so a startup EmitReset would
// tell consumers to throw out their world without sending the
// re-emission they need to rebuild it.
//
@@ -308,7 +308,7 @@ func (p *Poller) TriggerScanAll() (int, error) {
}
// TriggerReset performs a full reset for a watcher:
// 1. Clears the Redis stream and emits a reset event
// 1. Clears the file:events stream and emits a reset event
// 2. Clears the watcher's state
// 3. Performs a full scan (all files are "new")
func (p *Poller) TriggerReset(watcherID string) (int, error) {
internal/watchers/types.go
+1 -1
View File
@@ -21,7 +21,7 @@ type WatcherConfig struct {
// WatcherStatus includes runtime status in API responses
type WatcherStatus struct {
WatcherConfig
Active bool `json:"active"` // Polling goroutine running
Active bool `json:"active"` // Polling goroutine running
LastScan int64 `json:"lastScan"` // Unix ms timestamp
FileCount int `json:"fileCount"` // Files found in last scan
IsScanning bool `json:"isScanning"` // Currently scanning