diff --git a/README.md b/README.md index 4a37557..49c23a9 100644 --- a/README.md +++ b/README.md @@ -428,9 +428,8 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). | `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field` + FTS (`search.go`), UDL store (`udl.go`, `udl_admin.go`), stats. | | `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). | | `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. | -| `internal/events` | Boot-time republish of every field into `meta:events` (`meta_publisher.go`); live events are written by `internal/kv`. | | `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. | -| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). | +| `internal/schema` | Per-field schema, folded from counters the write path keeps exact (`idx:schema/…`). | | `internal/snapshot` | Snapshot export / import / wipe. | | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. | | `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). | @@ -447,10 +446,12 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). 5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false) 6. Construct API server (identity keystore migration, watcher, watchers, mounts, WebDAV) -7. Start API server + pollers; start MetaPublisher + schema Indexer, then - republish existing metadata onto meta:events in the background - (5k events per transaction) -8. Wait for SIGINT/SIGTERM, shutdown in reverse order +7. EnsureCounters — one-time build of the derived counters (record count, + per-field schema) on a store that predates them; blocking, ~1 min at 169k + records, then never again +8. Start API server + pollers (no republish: consumers that want the whole + state ask /api/events/meta for `0-0` and get a snapshot) +9. Wait for SIGINT/SIGTERM, shutdown in reverse order ``` ## Dependencies diff --git a/cmd/meta-core/main.go b/cmd/meta-core/main.go index a96d18d..6119c81 100644 --- a/cmd/meta-core/main.go +++ b/cmd/meta-core/main.go @@ -46,6 +46,16 @@ func main() { } log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion) + // Derived counters (record count, per-field schema) are kept exact by the + // write path. A store that predates them gets a one-time build here, + // before anything else writes. + started := time.Now() + if built, err := storageClient.EnsureCounters(); err != nil { + log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err) + } else if built { + log.Printf("[meta-core] Built derived counters in %s", time.Since(started).Round(time.Millisecond)) + } + // Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core // as metamesh.core — carrying the /urls block services locate us by — plus // its own metamesh.service/ nav entry. The payload is rebuilt from diff --git a/docs/api-mediated-access.md b/docs/api-mediated-access.md index dc05775..a1cc767 100644 --- a/docs/api-mediated-access.md +++ b/docs/api-mediated-access.md @@ -265,12 +265,16 @@ seen). The contract above holds unchanged: meta-share / meta-stremio before the switch still compare. - `meta:events` entries (`{type: set|del, key, ts}`) are appended by the storage engine **in the same transaction** as the `file:` key change — - `del` only when the key existed, `set` even for an unchanged value. The - Redis keyspace-notification hop (PSUBSCRIBE in `MetaPublisher`), which - could drop events when it lagged, is gone. -- The boot-time republish of every field into `meta:events` is kept - (meta-fuse rebuilds its VFS only by replaying from `0-0`), batched 5k - events per transaction. + `del` only when the key existed, `set` only when the value actually + changed (since v1.2.0; an unchanged write is a no-op). The Redis + keyspace-notification hop (PSUBSCRIBE in `MetaPublisher`), which could + drop events when it lagged, is gone. +- **`Last-Event-ID: 0` / `0-0` on `/api/events/meta` returns a snapshot** + (since v1.2.0): one `set` per existing record key, then the live tail from + the head captured before the snapshot. Only the last snapshot event carries + an `id:`. This replaced the boot-time republish of every field, which a + ~100k-entry stream could never hold. A trimmed (non-zero) cursor still gets + the `gap` event and a replay of what is retained. ### Heartbeats diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index b74fe58..2a3d59e 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -434,11 +434,10 @@ packages/meta-core/ │ ├── cid/ CID parsing + algorithm ranking │ ├── config/ env-driven configuration │ ├── discovery/ service registry + dead-service cleaner -│ ├── events/ boot-time meta:events republish │ ├── kv/ SQLite kv engine + event streams │ ├── leader/ LeaderInfoProvider (no election logic) │ ├── mounts/ rclone mount manager + handlers + stats -│ ├── schema/ live per-field schema indexer +│ ├── schema/ per-field schema from transactional counters │ ├── snapshot/ snapshot export / import / wipe │ ├── storage/ flat-key records + search + UUID + CID reverse index + schema gate │ ├── watcher/ scanner + dispatcher + midhash + state @@ -465,10 +464,11 @@ packages/meta-core/ 4. EnsureSchemaVersion (abort if the store layout is stale) 5. Start service discovery + dead-service cleaner 6. NewServer + Server.Start (boots watcher, mounts, WebDAV, API) -7. On first connection: start MetaPublisher + schema Indexer, - republish existing metadata onto meta:events -8. Wait for SIGINT/SIGTERM -9. Shutdown in reverse order +7. EnsureCounters (one-time build of the derived counters + on a store that predates them) +8. Rebuild FTS rows in the background if the FTS table is empty +9. Wait for SIGINT/SIGTERM +10. Shutdown in reverse order ``` ### Process management diff --git a/docs/metadata-storage-structure.md b/docs/metadata-storage-structure.md index 6af5304..7ddaf31 100644 --- a/docs/metadata-storage-structure.md +++ b/docs/metadata-storage-structure.md @@ -60,6 +60,9 @@ Everything is one string→string table. The full layout: | `meta-core:schema-version` | `"2"` | schema gate | | `idx:roots/` | `""` | set of every root (was the Redis SET `file:__index__`) | | `idx:fts/` | FTS rowid | the record's row in the FTS table | +| `idx:count/roots` | integer | number of roots — search's `total`, `/api/stats` records (v1.2.0) | +| `idx:schema/\x1f\x1f\x1f` | integer | per-field value-type counts behind `/api/schema` (v1.2.0) | +| `idx:count/ready` | `"1"` | marks the counters as built (absent → one-time build at boot) | | `dup:/` | `""` | a duplicate path (was the SET `file:/duplicates`) | | `udl:rec://` | JSON `{record,version,ts,value?}` | User Data Layer entry (was a HASH) | | `udl:idx:…\x1f` | `""` | UDL index sets, one key per member | @@ -291,12 +294,20 @@ high-water mark `evhwm:` (`internal/kv/stream.go`): | `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. | | `meta:events` | the storage engine (`internal/kv`) | `{type: set\|del, key, ts}` for every Put/Delete of a `file:` key, appended **in the same transaction** as the change. | -`del` is emitted only when the key existed; `set` is emitted even for an -unchanged value (Redis semantics, kept). Retention is ~100k entries, -trimmed every 10k appends. At boot `MetaPublisher` -(`internal/events/meta_publisher.go`) still republishes every field into -`meta:events` (meta-fuse rebuilds its VFS only by replaying from `0-0`), -batched 5k events per transaction. +`set` is emitted only for a **real** change: a Put of the value a key +already holds is a no-op — no row write, no index churn, no event (v1.2.0; +Redis published `set` even then, and with gateway hits re-persisted on +every repeated query most writes were exactly that). `del` is emitted only +when the key existed. Retention is ~100k entries, trimmed every 10k appends. + +There is **no boot-time republish** (v1.2.0). A consumer that wants the whole +current state asks `/api/events/meta` for `Last-Event-ID: 0` / `0-0` and gets +a **snapshot**: one `set` per existing record key, streamed from the kv +range, then the live tail from the stream head captured before the snapshot +(only the last snapshot event carries an `id:`, so a consumer that drops +mid-snapshot gets a fresh one). The old republish wrote every field into a +stream that retains only ~100k entries, so a consumer replaying it later +never saw most records — and it cost ~6 minutes on every boot. The Redis-era path — keyspace notifications drained by `MetaPublisher` through a PSUBSCRIBE / consumer-group hop — is gone (2026-10). @@ -371,7 +382,8 @@ curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats | UUIDv7 minting | `internal/storage/uuid.go` | | CID reverse index + aliases + canonical-CID derivation | `internal/storage/cid_resolution.go` | | Schema-version gate | `internal/storage/schema_version.go` | -| Boot-time `meta:events` republish | `internal/events/meta_publisher.go` | +| `meta:events` snapshot for `0`/`0-0` | `internal/api/sse_events.go` (`streamSnapshot`) | +| Derived counters (root count, per-field schema) | `internal/storage/client.go` (`onChange`), `internal/schema/indexer.go` | | Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` | | SSE event mirror | `internal/api/sse_events.go` | | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) | diff --git a/internal/api/schema_handlers.go b/internal/api/schema_handlers.go index 9591241..1549ee4 100644 --- a/internal/api/schema_handlers.go +++ b/internal/api/schema_handlers.go @@ -5,38 +5,44 @@ import ( "net/http" ) -// handleSchemaGet returns the per-field schema (recomputed from the store every -// few minutes; see schema.Indexer). +// handleSchemaGet returns the per-field schema, folded from counters the write +// path keeps exact (see schema.Indexer). func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) { if s.schemaIndexer == nil { http.Error(w, "schema indexer not running", http.StatusServiceUnavailable) return } - snap := s.schemaIndexer.Snapshot() + snap, err := s.schemaIndexer.Snapshot(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(snap) } -// handleSchemaRescan recomputes the schema from a full scan of the store and -// returns it. It no longer republishes every property into meta:events — the -// indexer reads the store itself — so a rescan floods no consumer. +// handleSchemaRescan rebuilds the schema counters from a full read of the +// records and returns the result. The counters are exact by construction, so +// this is a repair tool, not routine: it runs as one write transaction, and +// every other write waits for it (minutes on a large store). func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) { if s.schemaIndexer == nil { http.Error(w, "schema indexer not running", http.StatusServiceUnavailable) return } - - snap, err := s.schemaIndexer.Rebuild(r.Context()) + if err := s.storage.RebuildCounters(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + snap, err := s.schemaIndexer.Snapshot(r.Context()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - out := *snap - out.Source = "rescan" - + snap.Source = "rescan" w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "published_events": 0, - "schema": &out, + "schema": snap, }) } diff --git a/internal/api/server.go b/internal/api/server.go index 7717994..4088b63 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -9,7 +9,6 @@ import ( "github.com/gorilla/mux" "github.com/metazla/meta-core/internal/config" - "github.com/metazla/meta-core/internal/events" "github.com/metazla/meta-core/internal/identity" "github.com/metazla/meta-core/internal/leader" "github.com/metazla/meta-core/internal/meshdisco" @@ -36,7 +35,6 @@ type Server struct { watchersManager *watchers.Manager watchersPoller *watchers.Poller watchersHandlers *watchers.Handlers - metaPublisher *events.MetaPublisher schemaIndexer *schema.Indexer webdavHandler *webdav.Handler // challenges backs proof-of-possession on reveal/delete. In-process, so a @@ -107,8 +105,7 @@ func NewServer( // Create watchers poller if s.fileWatcher != nil { s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher) - // Pass republish callback to handlers (handles nil metaPublisher case) - s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata) + s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller) // Hydrate the in-memory state registry from stored tuples so // the first scan tick after boot can skip ComputeMidHash256 @@ -349,41 +346,22 @@ func (s *Server) Start() error { s.mountStatsPoller.Start() } - // Start meta publisher (if storage is connected). Live change events are - // written by the storage engine itself; the publisher owns the bulk - // republish below. + // Schema: served from counters the write path keeps exact; nothing runs. + // Change events need no publisher either — the storage engine appends them + // in the same transaction as each change, and a consumer that wants the + // whole state asks /api/events/meta for "0" and gets a snapshot (see + // streamSnapshot). The boot republish of every field is gone with it. if s.storage != nil && s.storage.IsConnected() { - s.metaPublisher = events.NewMetaPublisher(s.storage) - if err := s.metaPublisher.Start(); err != nil { - log.Printf("[API] Warning: failed to start meta publisher: %v", err) - } else { - // Schema indexer: per-field schema (type/hint/breakdown) from a - // periodic scan of the store. - s.schemaIndexer = schema.NewIndexer(s.storage.Store()) - if err := s.schemaIndexer.Start(); err != nil { - log.Printf("[API] Warning: failed to start schema indexer: %v", err) - s.schemaIndexer = nil + s.schemaIndexer = schema.NewIndexer(s.storage.Store()) + go func() { + started := time.Now() + if n, err := s.storage.RebuildSearchIfEmpty(); err != nil { + log.Printf("[API] Warning: search row rebuild failed: %v", err) + } else if n > 0 { + log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond)) } - - // Republish events for all existing metadata. Consumers that - // rebuild by replaying meta:events from the beginning (meta-fuse) - // depend on it. - go func() { - started := time.Now() - if n, err := s.storage.RebuildSearchIfEmpty(); err != nil { - log.Printf("[API] Warning: search row rebuild failed: %v", err) - } else if n > 0 { - log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond)) - } - - count, err := s.RepublishMetadata() - if err != nil { - log.Printf("[API] Warning: failed to republish metadata on startup: %v", err) - } else { - log.Printf("[API] Republished %d metadata events on startup", count) - } - }() - } + }() + go s.logWriteStats() } return nil @@ -410,20 +388,6 @@ func (s *Server) Stop() error { s.mountStatsPoller.Stop() } - // Stop schema indexer before the publisher so the consumer drains cleanly - if s.schemaIndexer != nil { - if err := s.schemaIndexer.Stop(); err != nil { - log.Printf("[API] Warning: failed to stop schema indexer: %v", err) - } - } - - // Stop meta publisher - if s.metaPublisher != nil { - if err := s.metaPublisher.Stop(); err != nil { - log.Printf("[API] Warning: failed to stop meta publisher: %v", err) - } - } - if s.server == nil { return nil } @@ -436,26 +400,22 @@ func (s *Server) Stop() error { return s.server.Shutdown(ctx) } -// RepublishMetadata republishes all existing metadata to the meta:events stream -// This is called on startup and after reset-all -func (s *Server) RepublishMetadata() (int, error) { - if s.metaPublisher == nil { - return 0, fmt.Errorf("meta publisher not initialized") - } - if s.storage == nil || !s.storage.IsConnected() { - return 0, fmt.Errorf("storage not connected") - } - - // Create the republish function that wraps storage methods - republishFunc := func() ([]string, func(string) (map[string]string, error), error) { - hashIDs, err := s.storage.GetAllHashIDs() - if err != nil { - return nil, nil, err +// logWriteStats reports, every 15 minutes, how many writes changed a value +// and how many found it already in place (kv skips those). The ratio is what +// the no-op skip buys on a permanent store that re-persists repeated hits. +func (s *Server) logWriteStats() { + ticker := time.NewTicker(15 * time.Minute) + defer ticker.Stop() + for range ticker.C { + store := s.storage.Store() + if store == nil { + return + } + changed, unchanged := store.WriteStats() + if total := changed + unchanged; total > 0 { + log.Printf("[Storage] writes since start: %d changed, %d unchanged skipped (%.0f%% no-op)", changed, unchanged, 100*float64(unchanged)/float64(total)) } - return hashIDs, s.storage.GetMetadataFlat, nil } - - return s.metaPublisher.RepublishAllMetadata(republishFunc) } // statusRecorder captures the response status so loggingMiddleware can stay diff --git a/internal/api/sse_events.go b/internal/api/sse_events.go index b063d0f..43b1dfb 100644 --- a/internal/api/sse_events.go +++ b/internal/api/sse_events.go @@ -11,6 +11,7 @@ import ( "time" "github.com/metazla/meta-core/internal/kv" + "github.com/metazla/meta-core/internal/storage" ) // SSE event stream endpoints for the file:events and meta:events streams. @@ -97,7 +98,22 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string } ctx := r.Context() - cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher) + lastID := r.Header.Get("Last-Event-ID") + var cursor string + if stream == storage.MetaEventsStream && (lastID == "0" || lastID == "0-0") { + // "From the beginning" on meta:events means "the whole current + // state": stream it from the records themselves, then the live tail. + head, err := streamSnapshot(ctx, store, w, flusher) + if err != nil { + if ctx.Err() == nil { + log.Printf("[SSE %s] snapshot failed: %v", stream, err) + } + return + } + cursor = head + } else { + cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher) + } lastWrite := time.Now() for { @@ -147,6 +163,54 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string } } +// streamSnapshot sends one `set` event per existing record key, then returns +// the stream position the live tail continues from. +// +// This is how a consumer that rebuilds its state from events (meta-fuse sends +// Last-Event-ID 0-0 on every start) gets ALL of it. The stream itself only +// retains ~100k entries, so replaying it never could; meta-core used to +// republish every field into it on boot instead, which took minutes and still +// lost everything past the retention window. +// +// The head is captured BEFORE the snapshot is read, so a change racing the +// snapshot appears in the tail too — re-applying a `set` is idempotent. Only +// the final snapshot event carries an `id:`, so a consumer that disconnects +// mid-snapshot still holds its old cursor and gets a fresh snapshot. +func streamSnapshot(ctx context.Context, store *kv.Store, w http.ResponseWriter, flusher http.Flusher) (string, error) { + head, err := store.LastID(ctx, storage.MetaEventsStream) + if err != nil { + return "", err + } + ts := strconv.FormatInt(time.Now().UnixMilli(), 10) + pending := "" + err = store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error { + for _, row := range rows { + if pending != "" { + writeSnapshotEvent(w, pending, ts, "") + } + pending = row.K + } + flusher.Flush() + return ctx.Err() + }) + if err != nil { + return "", err + } + if pending != "" { + writeSnapshotEvent(w, pending, ts, head) + flusher.Flush() + } + return head, nil +} + +func writeSnapshotEvent(w http.ResponseWriter, key, ts, id string) { + payload, _ := json.Marshal(map[string]string{"key": key, "ts": ts}) + if id != "" { + fmt.Fprintf(w, "id: %s\n", id) + } + fmt.Fprintf(w, "event: set\ndata: %s\n\n", payload) +} + // resolveStartCursor decides where to start reading. // - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only // entries appended from now on). Redis' "$" was re-resolved on every diff --git a/internal/api/sse_snapshot_test.go b/internal/api/sse_snapshot_test.go new file mode 100644 index 0000000..b5eacee --- /dev/null +++ b/internal/api/sse_snapshot_test.go @@ -0,0 +1,89 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +// syncRecorder is a ResponseRecorder safe to read while the handler writes. +type syncRecorder struct { + *httptest.ResponseRecorder + mu sync.Mutex +} + +func (r *syncRecorder) Write(b []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.ResponseRecorder.Write(b) +} + +func (r *syncRecorder) body() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.Body.String() +} + +// Last-Event-ID 0-0 on meta:events returns the whole current state — one set +// per record key, only the last carrying an id — then the live tail. +func TestMetaEventsSnapshotThenTail(t *testing.T) { + stor := newTestStorage(t) + s := &Server{storage: stor} + for i, id := range []string{"R1", "R2"} { + if err := stor.SetMetadataFlat(id, map[string]string{"title": "T", "n": string(rune('a' + i))}); err != nil { + t.Fatal(err) + } + } + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx) + req.Header.Set("Last-Event-ID", "0-0") + rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()} + done := make(chan struct{}) + go func() { s.handleEventsMeta(rec, req); close(done) }() + + waitFor := func(substr string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !strings.Contains(rec.body(), substr) { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %q in:\n%s", substr, rec.body()) + } + time.Sleep(20 * time.Millisecond) + } + } + waitFor(`"key":"file:R2/title"`) + if err := stor.SetProperty("R1", "title", "changed"); err != nil { + t.Fatal(err) + } + waitFor(`event: set`) + time.Sleep(300 * time.Millisecond) + cancel() + <-done + + body := rec.body() + for _, k := range []string{"file:R1/title", "file:R1/n", "file:R2/title", "file:R2/n"} { + if !strings.Contains(body, `"key":"`+k+`"`) { + t.Errorf("snapshot is missing %s", k) + } + } + events := strings.Split(strings.TrimSpace(body), "\n\n") + if len(events) != 5 { + t.Fatalf("want 4 snapshot events + 1 live event, got %d:\n%s", len(events), body) + } + for i, e := range events[:3] { + if strings.Contains(e, "id: ") { + t.Errorf("snapshot event %d carries an id: %q", i, e) + } + } + if !strings.HasPrefix(events[3], "id: ") || !strings.HasPrefix(events[4], "id: ") { + t.Fatalf("the last snapshot event and the live event must carry ids:\n%s", body) + } + if !strings.Contains(events[4], `"key":"file:R1/title"`) { + t.Fatalf("live tail did not follow the snapshot: %q", events[4]) + } +} diff --git a/internal/events/meta_publisher.go b/internal/events/meta_publisher.go deleted file mode 100644 index 29cfd43..0000000 --- a/internal/events/meta_publisher.go +++ /dev/null @@ -1,129 +0,0 @@ -package events - -import ( - "fmt" - "log" - "sync" - "time" - - "github.com/metazla/meta-core/internal/storage" -) - -const ( - // MetaEventsStream is the stream name for metadata events. - MetaEventsStream = storage.MetaEventsStream - - // MetaEventsMaxLen is the approximate cap on stream length. Bounded - // retention is a contract requirement for the SSE wire — see - // docs/api-mediated-access.md "Backing-store retention and lifecycle". - MetaEventsMaxLen = storage.MetaEventsMaxLen -) - -// republishBatch is how many events one republish transaction appends — -// large enough to amortise the commit, small enough that the writer is never -// held for long. -const republishBatch = 5_000 - -// MetaPublisher owns the meta:events stream's bulk republish. -// -// Live change events no longer pass through here: under Redis this subscribed -// to keyspace notifications and copied each into the stream, a hop that could -// drop events whenever the subscriber lagged. The storage engine now appends -// the event in the same transaction as the change (internal/kv), so a change -// and its event commit together. -type MetaPublisher struct { - client *storage.Client - running bool - mu sync.RWMutex -} - -// NewMetaPublisher creates a new MetaPublisher. -func NewMetaPublisher(client *storage.Client) *MetaPublisher { - return &MetaPublisher{client: client} -} - -// Start marks the publisher ready. The stream is NOT truncated on startup — -// the SSE wire contract requires Last-Event-ID resumption to survive a -// meta-core restart. Growth is bounded by MetaEventsMaxLen instead. -func (p *MetaPublisher) Start() error { - p.mu.Lock() - defer p.mu.Unlock() - p.running = true - log.Println("[MetaPublisher] Started") - return nil -} - -// Stop stops the publisher. -func (p *MetaPublisher) Stop() error { - p.mu.Lock() - defer p.mu.Unlock() - p.running = false - log.Println("[MetaPublisher] Stopped") - return nil -} - -// IsRunning returns true if the publisher is running. -func (p *MetaPublisher) IsRunning() bool { - p.mu.RLock() - defer p.mu.RUnlock() - return p.running -} - -// RepublishFunc is a function type for getting metadata that can be republished -type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error) - -// RepublishAllMetadata publishes a `set` event for every property of every -// record. Called on startup and via the API. -// -// It exists for consumers that rebuild their state by replaying meta:events -// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather -// than backfilling over HTTP. Appended in bounded batches, each its own -// transaction, so a multi-million-event republish never holds the writer. -func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) { - if !p.IsRunning() { - return 0, fmt.Errorf("publisher not running") - } - - hashIDs, getMetadata, err := republishFunc() - if err != nil { - return 0, fmt.Errorf("failed to get hash IDs: %w", err) - } - - log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs)) - - published := 0 - batch := make([]map[string]string, 0, republishBatch) - flush := func() error { - if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil { - return err - } - published += len(batch) - batch = batch[:0] - return nil - } - for _, hashID := range hashIDs { - metadata, err := getMetadata(hashID) - if err != nil || metadata == nil { - continue - } - ts := fmt.Sprintf("%d", time.Now().UnixMilli()) - for property := range metadata { - batch = append(batch, map[string]string{ - "type": "set", - "key": fmt.Sprintf("file:%s/%s", hashID, property), - "ts": ts, - }) - if len(batch) >= republishBatch { - if err := flush(); err != nil { - return published, fmt.Errorf("republish: %w", err) - } - } - } - } - if err := flush(); err != nil { - return published, fmt.Errorf("republish: %w", err) - } - - log.Printf("[MetaPublisher] Republished %d property events", published) - return published, nil -} diff --git a/internal/kv/kv.go b/internal/kv/kv.go index 8d1c161..9544c42 100644 --- a/internal/kv/kv.go +++ b/internal/kv/kv.go @@ -53,8 +53,10 @@ import ( "log" "net/url" "os" + "strconv" "strings" "sync" + "sync/atomic" "time" _ "modernc.org/sqlite" @@ -77,6 +79,11 @@ type Options struct { // Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS), // for derived indexes owned by the caller. Schema string + // OnChange is called inside the transaction for every key whose value + // actually changed (created, modified or deleted), with the old and new + // values — the hook derived counters hang off (Tx.AddCounter). It must not + // write kv rows itself. + OnChange func(tx *Tx, key, old string, hadOld bool, new string, hasNew bool) } // Store is an open database. @@ -93,6 +100,9 @@ type Store struct { // SQL on every Put was the single largest cost of a record write. stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt + // Write accounting: how many Puts changed a value vs found it unchanged. + puts, noops atomic.Int64 + writes chan *writeReq writeMu sync.Mutex // held by the writer while a group runs closeOnce sync.Once @@ -170,7 +180,7 @@ func Open(path string, opts Options) (*Store, error) { }{ {&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`}, {&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`}, - {&st.stDelete, `DELETE FROM kv WHERE k = ?`}, + {&st.stDelete, `DELETE FROM kv WHERE k = ? RETURNING v`}, {&st.stGet, `SELECT v FROM kv WHERE k = ?`}, {&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`}, } { @@ -470,9 +480,10 @@ func (s *Store) writer() { // group is the state shared by the requests of one commit. type group struct { - tx *sql.Tx - pending map[string]*streamPending - stmts map[*sql.Stmt]*sql.Stmt + tx *sql.Tx + pending map[string]*streamPending + stmts map[*sql.Stmt]*sql.Stmt + counters map[string]int64 } func (g *group) snapshot() map[string]streamPending { @@ -521,6 +532,9 @@ func (s *Store) runGroup(reqs []*writeReq) { if err == nil { err = tx.flushEvents() } + if err == nil { + g.mergeCounters(tx.counters) + } if err != nil { results[i] = err if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil { @@ -536,6 +550,11 @@ func (s *Store) runGroup(reqs []*writeReq) { return } } + if err := (&Tx{ctx: ctx, s: s, g: g}).flushCounters(); err != nil { + _ = sqlTx.Rollback() + fail(err) + return + } if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil { _ = sqlTx.Rollback() fail(err) @@ -554,10 +573,11 @@ func (s *Store) runGroup(reqs []*writeReq) { // Tx is one Update's view of the write transaction. Its reads see its own // writes (and those of earlier requests in the same group). type Tx struct { - ctx context.Context - s *Store - g *group - events []map[string]string + ctx context.Context + s *Store + g *group + events []map[string]string + counters map[string]int64 } func (t *Tx) sqlTx() *sql.Tx { return t.g.tx } @@ -610,18 +630,34 @@ func (t *Tx) QueryRow(query string, args ...any) *sql.Row { return t.sqlTx().QueryRowContext(t.ctx, query, args...) } -// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis -// keyspace notifications did. +// Put sets k = v. A no-op — no row write, no index churn, no event — when k +// already holds v. (Redis keyspace notifications published `set` even for an +// unchanged value; with meta-core persisting the same gateway hits again on +// every repeated query, those no-op writes were most of the write load.) func (t *Tx) Put(k, v string) error { - if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil { - return fmt.Errorf("put %q: %w", k, err) + _, err := t.Set(k, v) + return err +} + +// Set is Put that reports whether the value changed. +func (t *Tx) Set(k, v string) (bool, error) { + old, had, err := t.Get(t.ctx, k) + if err != nil { + return false, fmt.Errorf("put %q: %w", k, err) } - t.event("set", k) - return nil + if had && old == v { + t.s.noops.Add(1) + return false, nil + } + if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil { + return false, fmt.Errorf("put %q: %w", k, err) + } + t.s.puts.Add(1) + t.changed("set", k, old, had, v, true) + return true, nil } // PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote. -// Publishes nothing when the key already existed (nothing changed). func (t *Tx) PutIfAbsent(k, v string) (bool, error) { res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v) if err != nil { @@ -629,7 +665,7 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) { } n, _ := res.RowsAffected() if n > 0 { - t.event("set", k) + t.changed("set", k, "", false, v, true) } return n > 0, nil } @@ -637,15 +673,16 @@ func (t *Tx) PutIfAbsent(k, v string) (bool, error) { // Delete removes k. Reports whether it existed; publishes `del` only then, // as Redis did. func (t *Tx) Delete(k string) (bool, error) { - res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k) + var old string + err := t.stmt(t.s.stDelete).QueryRowContext(t.ctx, k).Scan(&old) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } if err != nil { return false, fmt.Errorf("delete %q: %w", k, err) } - n, _ := res.RowsAffected() - if n > 0 { - t.event("del", k) - } - return n > 0, nil + t.changed("del", k, old, true, "", false) + return true, nil } // DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and @@ -663,26 +700,103 @@ func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) { sel += ` ORDER BY k LIMIT ?` args = append(args, limit) } - rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...) + rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k, v`, args...) if err != nil { return 0, fmt.Errorf("delete prefix %q: %w", prefix, err) } - var keys []string + var gone []KV for rows.Next() { - var k string - if err := rows.Scan(&k); err != nil { + var kv KV + if err := rows.Scan(&kv.K, &kv.V); err != nil { rows.Close() return 0, err } - keys = append(keys, k) + gone = append(gone, kv) } if err := rows.Close(); err != nil { return 0, err } - for _, k := range keys { - t.event("del", k) + for _, kv := range gone { + t.changed("del", kv.K, kv.V, true, "", false) } - return int64(len(keys)), nil + return int64(len(gone)), nil +} + +// changed records one real change: the event (when in scope) and the hook. +func (t *Tx) changed(op, k, old string, hadOld bool, v string, hasNew bool) { + t.event(op, k) + if t.s.opts.OnChange != nil { + t.s.opts.OnChange(t, k, old, hadOld, v, hasNew) + } +} + +// AddCounter adds delta to the integer counter stored at key. Deltas are +// summed in memory and written once per commit (a hot counter touched by every +// record in a group costs one row write, not one per record); they are +// discarded with the request if it fails. A counter that reaches zero is +// deleted. +func (t *Tx) AddCounter(key string, delta int64) { + if delta == 0 { + return + } + if t.counters == nil { + t.counters = make(map[string]int64) + } + t.counters[key] += delta +} + +func (g *group) mergeCounters(c map[string]int64) { + if len(c) == 0 { + return + } + if g.counters == nil { + g.counters = make(map[string]int64, len(c)) + } + for k, d := range c { + g.counters[k] += d + } +} + +// flushCounters applies the group's counter deltas, bypassing Put (counters +// are bookkeeping: no events, no hook). +func (t *Tx) flushCounters() error { + for k, d := range t.g.counters { + if d == 0 { + continue + } + cur, _, err := t.Get(t.ctx, k) + if err != nil { + return err + } + n, _ := strconv.ParseInt(cur, 10, 64) + n += d + if n <= 0 { + if _, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k = ?`, k); err != nil { + return fmt.Errorf("counter %s: %w", k, err) + } + continue + } + if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, strconv.FormatInt(n, 10)); err != nil { + return fmt.Errorf("counter %s: %w", k, err) + } + } + return nil +} + +// Counter reads a committed counter (0 when absent). +func (s *Store) Counter(ctx context.Context, key string) (int64, bool, error) { + v, ok, err := s.Get(ctx, key) + if err != nil || !ok { + return 0, ok, err + } + n, err := strconv.ParseInt(v, 10, 64) + return n, true, err +} + +// WriteStats reports how many Puts changed a value and how many found it +// already in place, since open. +func (s *Store) WriteStats() (changed, unchanged int64) { + return s.puts.Load(), s.noops.Load() } func (t *Tx) event(op, k string) { diff --git a/internal/kv/kv_test.go b/internal/kv/kv_test.go index 865d1c4..4f344cc 100644 --- a/internal/kv/kv_test.go +++ b/internal/kv/kv_test.go @@ -44,20 +44,21 @@ func events(t *testing.T, s *Store) []Entry { } // The replacement for keyspace notifications: a change and its event commit -// together, with Redis' semantics — `set` even for an unchanged value, `del` -// only for a key that existed, and nothing for keys outside the event scope. +// together. `set` only for a real change (an unchanged value is a no-op: no +// write, no event), `del` only for a key that existed, nothing for keys outside +// the event scope. func TestChangesPublishEventsWithRedisSemantics(t *testing.T) { s := openTest(t, "") update(t, s, func(tx *Tx) error { _ = tx.Put("file:A/title", "x") - _ = tx.Put("file:A/title", "x") // unchanged value still publishes + _ = tx.Put("file:A/title", "x") // unchanged: no write, no event _ = tx.Put("cid:bafk", "A") // outside the event scope _, _ = tx.Delete("file:A/missing") _, _ = tx.Delete("file:A/title") return nil }) got := events(t, s) - want := []string{"set file:A/title", "set file:A/title", "del file:A/title"} + want := []string{"set file:A/title", "del file:A/title"} if len(got) != len(want) { t.Fatalf("events = %+v, want %v", got, want) } @@ -312,3 +313,50 @@ func TestGroupCommitIsolatesFailures(t *testing.T) { } } } + +// The change hook sees real changes only, with old and new values, and +// counters follow the request's fate (a failed request's deltas are dropped). +func TestOnChangeAndCounters(t *testing.T) { + type change struct{ k, old, new string } + var seen []change + s, err := Open(filepath.Join(t.TempDir(), "kv.db"), Options{ + OnChange: func(tx *Tx, k, old string, hadOld bool, v string, hasNew bool) { + seen = append(seen, change{k, old, v}) + if hasNew { + tx.AddCounter("count/"+v, 1) + } + if hadOld { + tx.AddCounter("count/"+old, -1) + } + }, + }) + if err != nil { + t.Fatal(err) + } + defer s.Close() + update(t, s, func(tx *Tx) error { + _ = tx.Put("a", "x") + _ = tx.Put("b", "x") + _ = tx.Put("a", "x") // no-op + _ = tx.Put("a", "y") + return nil + }) + _ = s.Update(context.Background(), func(tx *Tx) error { + _ = tx.Put("c", "x") + return errors.New("fail") + }) + update(t, s, func(tx *Tx) error { _, err := tx.DeletePrefix("b", 0); return err }) + + if len(seen) != 5 { // a=x, b=x, a x→y, c=x (rolled back, hook still ran), del b; stats count attempts + t.Fatalf("changes = %+v", seen) + } + for k, want := range map[string]int64{"count/x": 0, "count/y": 1} { + got, _, err := s.Counter(context.Background(), k) + if err != nil || got != want { + t.Errorf("%s = %d (err %v), want %d", k, got, err, want) + } + } + if changed, unchanged := s.WriteStats(); changed != 4 || unchanged != 1 { + t.Errorf("write stats = %d changed / %d unchanged", changed, unchanged) + } +} diff --git a/internal/schema/indexer.go b/internal/schema/indexer.go index 44afbef..f4fc83e 100644 --- a/internal/schema/indexer.go +++ b/internal/schema/indexer.go @@ -2,10 +2,9 @@ package schema import ( "context" - "log" "regexp" + "strconv" "strings" - "sync" "time" "github.com/metazla/meta-core/internal/kv" @@ -13,11 +12,23 @@ import ( var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`) -// rebuildInterval is how often the schema is recomputed in the background. -// It describes the *shape* of the data (which fields exist, their types), which -// drifts slowly; a few minutes of staleness is invisible to the editor that -// reads it, and POST /api/schema/rescan recomputes on demand. -const rebuildInterval = 10 * time.Minute +// CounterPrefix is where the schema lives: one counter per (field, key hint, +// primitive, hint), maintained by the storage write path in the same +// transaction as the change (storage's OnChange hook → CounterKey). +const CounterPrefix = "idx:schema/" + +const counterSep = "\x1f" + +// CounterKey is the counter a record key holding value contributes to, or +// ok=false for keys that are not record properties. +func CounterKey(recordKey, value string) (string, bool) { + fieldPath, keyHint, ok := parseFieldPath(recordKey) + if !ok { + return "", false + } + prim, hint := ClassifyValue(value, true) + return CounterPrefix + fieldPath + counterSep + string(keyHint) + counterSep + string(prim) + counterSep + string(hint), true +} // fieldState is the per-field accumulator. type fieldState struct { @@ -37,123 +48,63 @@ func newFieldState() *fieldState { } } -// Indexer maintains the per-field schema by scanning the store. +// Indexer serves the per-field schema. // -// It used to follow meta:events and keep, per field, a map of EVERY record key -// it had seen (so a re-set or delete could undo that key's contribution). At -// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident -// memory once the search copy was gone — and it only worked if every event was -// replayed into it, which is half of why meta-core republished every field on -// boot. A periodic streaming scan holds one accumulator per FIELD instead and -// is exact by construction. +// The schema is a set of exact counters the write path keeps current in the +// same transaction as each change (see CounterKey), so serving it is one range +// read and nothing runs in the background. Two earlier shapes were dropped: a +// consumer of meta:events that remembered every record key it had seen (~2 GB +// at 6.3M keys), then a full rescan of the store every 10 minutes (~80 s of +// CPU each time, whether or not anyone looked). type Indexer struct { store *kv.Store - - mu sync.RWMutex - snap *SchemaResponse - - rebuildMu sync.Mutex - - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - runMu sync.Mutex - running bool } // NewIndexer creates an Indexer over store. func NewIndexer(store *kv.Store) *Indexer { - ctx, cancel := context.WithCancel(context.Background()) - return &Indexer{store: store, ctx: ctx, cancel: cancel} + return &Indexer{store: store} } -// Start computes the schema in the background now and every rebuildInterval. -func (i *Indexer) Start() error { - i.runMu.Lock() - defer i.runMu.Unlock() - if i.running { - return nil - } - i.running = true - i.wg.Add(1) - go func() { - defer i.wg.Done() - ticker := time.NewTicker(rebuildInterval) - defer ticker.Stop() - for { - if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil { - log.Printf("[SchemaIndexer] rebuild failed: %v", err) - } - select { - case <-i.ctx.Done(): - return - case <-ticker.C: - } - } - }() - log.Println("[SchemaIndexer] Started - scanning the store") - return nil -} +// Start and Stop exist for the server's lifecycle; there is nothing to run. +func (i *Indexer) Start() error { return nil } +func (i *Indexer) Stop() error { return nil } -// Stop terminates the background rebuilds. -func (i *Indexer) Stop() error { - i.runMu.Lock() - if !i.running { - i.runMu.Unlock() - return nil - } - i.running = false - i.runMu.Unlock() - - i.cancel() - i.wg.Wait() - log.Println("[SchemaIndexer] Stopped") - return nil -} - -// Rebuild recomputes the schema from every record key and publishes it. One -// rebuild runs at a time; a concurrent caller waits and gets the fresh result. -func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) { - i.rebuildMu.Lock() - defer i.rebuildMu.Unlock() - - started := time.Now() +// Snapshot reads the counters and folds them into the per-field schema. +func (i *Indexer) Snapshot(ctx context.Context) (*SchemaResponse, error) { fields := make(map[string]*fieldState) - err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error { + err := i.store.ScanPrefix(ctx, CounterPrefix, 5000, func(rows []kv.KV) error { for _, row := range rows { - fieldPath, keyHint, ok := parseFieldPath(row.K) - if !ok { + parts := strings.Split(row.K[len(CounterPrefix):], counterSep) + if len(parts) != 4 { continue } - fs := fields[fieldPath] + n, err := strconv.Atoi(row.V) + if err != nil || n <= 0 { + continue + } + fs := fields[parts[0]] if fs == nil { fs = newFieldState() - fields[fieldPath] = fs + fields[parts[0]] = fs } - if keyHint != "" { + if parts[1] != "" { fs.languageKeyed = true } - prim, hint := ClassifyValue(row.V, true) - incContribution(fs, labelPair{prim: prim, hint: hint}) + incContribution(fs, labelPair{prim: Primitive(parts[2]), hint: Hint(parts[3])}, n) } - return ctx.Err() + return nil }) if err != nil { return nil, err } - snap := &SchemaResponse{ Fields: make(map[string]*FieldSchema, len(fields)), GeneratedAt: time.Now().UTC(), - Source: "scan", + Source: "live", } for fp, fs := range fields { snap.Fields[fp] = computeFieldSchema(fs) } - i.mu.Lock() - i.snap = snap - i.mu.Unlock() - log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond)) return snap, nil } @@ -191,31 +142,21 @@ func parseFieldPath(key string) (string, KeyHint, bool) { return property, "", true } -func incContribution(fs *fieldState, lp labelPair) { +// incContribution adds n values of one (primitive, hint) to a field. +func incContribution(fs *fieldState, lp labelPair, n int) { label := breakdownLabel(lp.prim, lp.hint) - fs.counts[label]++ + fs.counts[label] += n if lp.prim == PrimUndefined { - fs.undefinedCount++ + fs.undefinedCount += n return } - fs.typedCount++ - fs.primitives[lp.prim]++ + fs.typedCount += n + fs.primitives[lp.prim] += n if lp.hint != "" { - fs.hintCounts[lp.hint]++ + fs.hintCounts[lp.hint] += n } } -// Snapshot returns the latest computed schema. Before the first scan finishes -// it is empty with Source "pending". -func (i *Indexer) Snapshot() *SchemaResponse { - i.mu.RLock() - defer i.mu.RUnlock() - if i.snap == nil { - return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"} - } - return i.snap -} - func computeFieldSchema(fs *fieldState) *FieldSchema { out := &FieldSchema{ Breakdown: make(map[string]int, len(fs.counts)), diff --git a/internal/schema/indexer_test.go b/internal/schema/indexer_test.go deleted file mode 100644 index d1e5c7b..0000000 --- a/internal/schema/indexer_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package schema - -import ( - "context" - "path/filepath" - "testing" - - "github.com/metazla/meta-core/internal/kv" -) - -func TestRebuildDerivesTypesFromTheStore(t *testing.T) { - store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{}) - if err != nil { - t.Fatal(err) - } - defer store.Close() - if err := store.Update(context.Background(), func(tx *kv.Tx) error { - for k, v := range map[string]string{ - "file:A/sizeByte": "100", - "file:B/sizeByte": "200", - "file:A/title/eng": "One", - "file:B/title/fre": "Deux", - "file:A/mixed": "1", - "file:B/mixed": "x", - "cid:bafknotrecord": "A", - } { - if err := tx.Put(k, v); err != nil { - return err - } - } - return nil - }); err != nil { - t.Fatal(err) - } - - ix := NewIndexer(store) - if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 { - t.Fatalf("before a scan: %+v", got) - } - snap, err := ix.Rebuild(context.Background()) - if err != nil { - t.Fatal(err) - } - if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 { - t.Fatalf("sizeByte = %+v", f) - } - if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode { - t.Fatalf("title = %+v (want language-keyed)", f) - } - if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed { - t.Fatalf("mixed = %+v", f) - } - if len(snap.Fields) != 3 { - t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields) - } -} diff --git a/internal/storage/client.go b/internal/storage/client.go index 09fbf02..7d411de 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -13,6 +13,7 @@ import ( "time" "github.com/metazla/meta-core/internal/kv" + "github.com/metazla/meta-core/internal/schema" ) // Key layout (everything is one string→string table, see internal/kv): @@ -21,6 +22,7 @@ import ( // cid: reverse index → // idx:roots/ the set of every root (was the Redis SET file:__index__) // idx:fts/ rowid of the record's row in the FTS table +// idx:count/…, idx:schema/… derived counters (root count, per-field schema), see onChange // dup:/ a duplicate path of the record (was the SET file:/duplicates) // udl:… User Data Layer, see udl.go // ev:/…, evhwm:… event streams, see internal/kv/stream.go @@ -80,6 +82,7 @@ func (c *Client) Connect(path string) error { ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), Readers: envInt("META_CORE_DB_READERS", 8), Schema: searchSchema, + OnChange: onChange, }) if err != nil { return fmt.Errorf("open %s: %w", path, err) @@ -185,6 +188,101 @@ func isInternalKey(key string) bool { strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS) } +// Derived counters, maintained by onChange in the same transaction as the +// change they count (kv.Tx.AddCounter): the root count behind search's `total` +// and /api/stats, and the per-field schema behind /api/schema. +const ( + rootsCountKey = "idx:count/roots" + countersReady = "idx:count/ready" + counterRebuild = 5000 +) + +// onChange keeps the derived counters exact. It sees only real changes (kv +// skips unchanged Puts) with the old and new values, so a modified field moves +// one count from its old label to its new one. +func onChange(tx *kv.Tx, k, old string, hadOld bool, v string, hasNew bool) { + if strings.HasPrefix(k, rootsNS) { + switch { + case hasNew && !hadOld: + tx.AddCounter(rootsCountKey, 1) + case hadOld && !hasNew: + tx.AddCounter(rootsCountKey, -1) + } + return + } + if !strings.HasPrefix(k, recordNS) { + return + } + if hadOld { + if ck, ok := schema.CounterKey(k, old); ok { + tx.AddCounter(ck, -1) + } + } + if hasNew { + if ck, ok := schema.CounterKey(k, v); ok { + tx.AddCounter(ck, 1) + } + } +} + +// EnsureCounters builds the derived counters when they have never been built — +// a store created before they existed, or one restored from outside meta-core. +// Blocking and single-transaction (writes wait for it; ~1–2 min at millions of +// keys, once), because a build racing live deltas would double-count. +func (c *Client) EnsureCounters() (bool, error) { + s, err := c.kv() + if err != nil { + return false, err + } + ctx, cancel := opCtx() + _, ready, err := s.Get(ctx, countersReady) + cancel() + if err != nil || ready { + return false, err + } + return true, c.RebuildCounters() +} + +// RebuildCounters recomputes every derived counter from the records, in one +// transaction (writers wait for it). +func (c *Client) RebuildCounters() error { + s, err := c.kv() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + return s.Update(ctx, func(tx *kv.Tx) error { + for _, p := range []string{schema.CounterPrefix, rootsCountKey} { + if _, err := tx.DeletePrefix(p, 0); err != nil { + return err + } + } + roots, err := tx.Count(tx.Context(), rootsNS) + if err != nil { + return err + } + tx.AddCounter(rootsCountKey, roots) + after := "" + for { + rows, err := tx.Range(tx.Context(), recordNS, after, counterRebuild) + if err != nil { + return err + } + for _, row := range rows { + if ck, ok := schema.CounterKey(row.K, row.V); ok { + tx.AddCounter(ck, 1) + } + } + if len(rows) < counterRebuild { + break + } + after = rows[len(rows)-1].K + } + return tx.Put(countersReady, "1") + }) +} + // readRecord returns a record's flat fields (nil when it has none). func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) { prefix := recordPrefix(hashID) @@ -210,17 +308,19 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error { return fmt.Errorf("empty hash id") } prefix := recordPrefix(hashID) - touchesSearch := false + searchChanged := false for field, value := range metadata { if isReservedFieldName(field) { continue } - if err := tx.Put(prefix+field, value); err != nil { + changed, err := tx.Set(prefix+field, value) + if err != nil { return err } - touchesSearch = touchesSearch || affectsHaystack(field) + searchChanged = searchChanged || (changed && affectsHaystack(field)) } - if err := tx.Put(rootKey(hashID), ""); err != nil { + newRoot, err := tx.Set(rootKey(hashID), "") + if err != nil { return err } for field := range metadata { @@ -230,10 +330,12 @@ func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error { } } } - if touchesSearch { + // The FTS row is rewritten only when the haystack can have changed — a + // re-persisted record whose values are all in place costs reads only. + if searchChanged || newRoot { return refreshSearchTx(tx, hashID) } - return ensureSearchRowTx(tx, hashID) + return nil } // GetMetadataFlat retrieves all metadata for a file as a flat map. diff --git a/internal/storage/counters_test.go b/internal/storage/counters_test.go new file mode 100644 index 0000000..1750b21 --- /dev/null +++ b/internal/storage/counters_test.go @@ -0,0 +1,92 @@ +package storage + +import ( + "context" + "encoding/json" + "testing" + + "github.com/metazla/meta-core/internal/schema" +) + +func schemaJSON(t *testing.T, c *Client) string { + t.Helper() + snap, err := schema.NewIndexer(c.Store()).Snapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + snap.GeneratedAt = snap.GeneratedAt.Truncate(0).UTC() + b, _ := json.Marshal(snap.Fields) + return string(b) +} + +// The counters behind /api/schema and search's `total` are maintained by the +// write path. Whatever sequence of writes, modifications and deletions runs, +// they must equal what a full rebuild from the records computes. +func TestCountersStayExactAcrossWrites(t *testing.T) { + c := newTestClient(t) + seed(t, c, map[string]map[string]string{ + "R1": {"title": "One", "sizeByte": "100", "title/eng": "One"}, + "R2": {"title": "Two", "sizeByte": "x", "genres/Action": "true"}, + "R3": {"title": "Three", "sizeByte": "300"}, + }) + if err := c.SetProperty("R2", "sizeByte", "200"); err != nil { // label string → int + t.Fatal(err) + } + if err := c.SetProperty("R1", "title", "One"); err != nil { // unchanged: no delta + t.Fatal(err) + } + if err := c.DeleteProperty("R1", "title/eng"); err != nil { + t.Fatal(err) + } + if _, err := c.DeleteMetadata("R3"); err != nil { + t.Fatal(err) + } + + if n, err := c.CountRoots(); err != nil || n != 2 { + t.Fatalf("CountRoots = %d (err %v), want 2", n, err) + } + incremental := schemaJSON(t, c) + snap, _ := schema.NewIndexer(c.Store()).Snapshot(context.Background()) + if f := snap.Fields["sizeByte"]; f == nil || f.Type != schema.PrimInt || f.Breakdown["int"] != 2 { + t.Fatalf("sizeByte = %+v, want 2 ints", f) + } + if _, ok := snap.Fields["title"]; !ok { + t.Fatalf("title missing: %v", snap.Fields) + } + + if err := c.RebuildCounters(); err != nil { + t.Fatal(err) + } + if rebuilt := schemaJSON(t, c); rebuilt != incremental { + t.Fatalf("incremental counters drifted from a rebuild:\n incremental %s\n rebuilt %s", incremental, rebuilt) + } + if n, _ := c.CountRoots(); n != 2 { + t.Fatalf("CountRoots after rebuild = %d", n) + } +} + +func TestEnsureCountersBuildsOnce(t *testing.T) { + c := newTestClient(t) + seed(t, c, map[string]map[string]string{"R1": {"title": "One"}}) + built, err := c.EnsureCounters() + if err != nil || !built { + t.Fatalf("first EnsureCounters built=%v err=%v", built, err) + } + if built, _ := c.EnsureCounters(); built { + t.Fatal("second EnsureCounters rebuilt again") + } +} + +// A re-persisted record with nothing new writes nothing and publishes nothing. +func TestUnchangedRecordWriteIsANoOp(t *testing.T) { + c := newTestClient(t) + rec := map[string]string{"title": "Naruto", "fileType": "video", "cids/bafkreiabc": "true"} + seed(t, c, map[string]map[string]string{"R1": rec}) + before, _ := c.GetStreamLength(MetaEventsStream) + if err := c.SetMetadataFlat("R1", rec); err != nil { + t.Fatal(err) + } + if after, _ := c.GetStreamLength(MetaEventsStream); after != before { + t.Fatalf("re-persisting an unchanged record published %d events", after-before) + } +} diff --git a/internal/storage/search.go b/internal/storage/search.go index 304a28d..336c746 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -211,21 +211,36 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f ctx, cancel := opCtx() defer cancel() - driver, probes, err := c.planCandidates(ctx, s, query, filters) + plan, err := c.planCandidates(ctx, s, query, filters) if err != nil { return nil, err } page := max(limit*2, 500) hits := make([]SearchHit, 0, limit) after := "" - for len(hits) < limit { - ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page) - if err != nil { - return nil, err + for pos := 0; len(hits) < limit; { + var ids []string + if plan.walk { + rows, err := s.Range(ctx, rootsNS, after, page) + if err != nil { + return nil, err + } + for _, row := range rows { + ids = append(ids, row.K[len(rootsNS):]) + } + if len(rows) > 0 { + after = rows[len(rows)-1].K + } + } else { + ids = plan.ordered[pos:min(pos+page, len(plan.ordered))] + pos += len(ids) + } + if len(ids) == 0 { + break } survivors := ids - if len(probes) > 0 && len(ids) > 0 { - if survivors, err = probeIDs(ctx, s, ids, probes); err != nil { + if len(plan.probes) > 0 { + if survivors, err = probeIDs(ctx, s, ids, plan.probes); err != nil { return nil, err } } @@ -245,30 +260,10 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f if len(ids) < page { break } - after = ids[len(ids)-1] } return hits, nil } -// queryIDs returns one page of the driver's candidates, in rid order. -func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) { - q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?` - rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...) - if err != nil { - return nil, fmt.Errorf("search candidates: %w", err) - } - defer rows.Close() - var out []string - for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - return nil, err - } - out = append(out, id) - } - return out, rows.Err() -} - // probeSQL applies the non-driving filters to one page of candidate rids // (passed as a JSON array) — each an EXISTS over the candidate's own key range. func probeSQL(probes []candidatePart) (string, []any) { @@ -304,7 +299,7 @@ func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidate if err := rows.Err(); err != nil { return nil, err } - out := ids[:0:0] + out := make([]string, 0, len(ids)) for _, id := range ids { if pass[id] { out = append(out, id) @@ -324,25 +319,40 @@ type candidatePart struct { existsArgs []any } -// estimateCap bounds the count used to pick the driving part: it only has to -// tell a 50-row part from a 100k-row one. -const estimateCap = 1000 +// estimateCap bounds the count used to rank parts by selectivity: past it, a +// part is "broad" and only ties with other broad parts are left to resolve. +var estimateCap = 50_000 -// planCandidates picks how to narrow a search: one DRIVING part (the most -// selective, by a capped count) that lists candidate rids in order, plus PROBES -// — every other filter, checked as an EXISTS over each candidate's own key -// range (a primary-key range read of ~40 rows). +// estimateCapForTest sets estimateCap and returns the previous value. +func estimateCapForTest(n int) int { old := estimateCap; estimateCap = n; return old } + +// searchPlan is how one search walks its candidates. +type searchPlan struct { + ordered []string // the driver's rids, sorted (when !walk) + walk bool // no part narrows: walk idx:roots in order + probes []candidatePart // filters checked per page as EXISTS +} + +// planCandidates picks how to narrow a search. // -// Search pages the driver in Go and probes one page at a time, so a broad -// query stops as soon as `limit` records verify. Two earlier shapes were -// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets -// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s) -// and probing inside the driver query (SQLite evaluates the probe on every -// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`, -// both ~100k, 10 s). Free text that does not drive is dropped from SQL -// entirely — the verifier still applies it, and dropping a condition only -// widens the superset. -func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) { +// A capped count ranks the parts; the SMALLEST is collected once, sorted in Go +// and paged from memory, and every other filter becomes a probe — an EXISTS +// over each candidate's own key range, run one page at a time, so a search +// stops as soon as `limit` records verify. The driver is collected in full even +// when every part is broad: that costs O(part) — at most every root, ~0.3 s at +// 169k — whereas walking idx:roots in hashID order (tried) costs O(corpus) when +// matches are sparse early in that order, and on watch.nsl.sh they are: the +// first 500 roots held zero episodes, so `contentKind:episode` walked ~3 s. +// Only a search with nothing to narrow walks the roots, where every root +// matches. +// +// Also rejected, measured on watch.nsl.sh: INTERSECTing fully materialised +// sets (5.7 s for `naruto` + `fileType:video`); probing inside a driver query +// re-sorted per page (SQLite evaluates the probe on every driver row before +// ORDER BY … LIMIT can stop: 10 s for `1080p` + `fileType:video`). Free text +// that does not drive is dropped from SQL — the verifier still applies it, and +// dropping a condition only widens the superset. +func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (searchPlan, error) { var parts []candidatePart if p, ok := textCandidates(query); ok { parts = append(parts, p) @@ -353,7 +363,7 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, } } if len(parts) == 0 { - return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil + return searchPlan{walk: true}, nil } driver := 0 @@ -361,40 +371,67 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, best := int64(-1) for i, p := range parts { var n int64 - q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)` + q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap+1) + `)` if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil { - return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err) + return searchPlan{}, fmt.Errorf("estimate candidates: %w", err) } if best < 0 || n < best { best, driver = n, i } + if n == 0 { + break // nothing can match + } } } - var probes []candidatePart + ids, err := collectIDs(ctx, s, parts[driver], 0) + if err != nil { + return searchPlan{}, err + } + sort.Strings(ids) + plan := searchPlan{ordered: ids} for i, p := range parts { if i != driver && p.exists != "" { - probes = append(probes, p) + plan.probes = append(plan.probes, p) } } - return parts[driver], probes, nil + return plan, nil } -// candidateSQL is planCandidates without the store: text plus filters, the -// first part driving. Used by tests that inspect the plan. +// collectIDs reads the distinct rids of one part (all of them when limit <= 0). +func collectIDs(ctx context.Context, s *kv.Store, p candidatePart, limit int) ([]string, error) { + q := `SELECT DISTINCT rid FROM (` + p.sql + `)` + args := append([]any{}, p.args...) + if limit > 0 { + q += ` LIMIT ?` + args = append(args, limit) + } + rows, err := s.DB().QueryContext(ctx, q, args...) + if err != nil { + return nil, fmt.Errorf("search candidates: %w", err) + } + defer rows.Close() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// candidateSQL is the first part's SQL, for tests that inspect the plan. func candidateSQL(query string, filters []SearchFilter) (string, []any) { - var parts []candidatePart if p, ok := textCandidates(query); ok { - parts = append(parts, p) + return p.sql, p.args } for _, f := range filters { if p, ok := filterCandidates(f); ok { - parts = append(parts, p) + return p.sql, p.args } } - if len(parts) == 0 { - return "", nil - } - return parts[0].sql, parts[0].args + return "", nil } // textCandidates narrows by free text: every whitespace token must be a @@ -480,7 +517,7 @@ func filterCandidates(f SearchFilter) (candidatePart, bool) { // As a driver: one indexed SELECT per condition, joined with UNION ALL (an // OR inside one SELECT would defeat the partial index; plain UNION would // dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped - // count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT. + // count over fileType:video on watch.nsl.sh). collectIDs applies DISTINCT. base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND ` selects := make([]string, len(conds)) for i, cond := range conds { diff --git a/internal/storage/search_test.go b/internal/storage/search_test.go index 0a92f86..21b0bb8 100644 --- a/internal/storage/search_test.go +++ b/internal/storage/search_test.go @@ -294,13 +294,14 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) { } seed(t, c, recs) filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}} - driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters) + plan, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters) if err != nil { t.Fatal(err) } - if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 { - t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes)) + if plan.walk || len(plan.ordered) != 1 || len(plan.probes) != 1 { + t.Fatalf("the rare text should drive (1 rid) with the filter as a probe, got %+v", plan) } + probes := plan.probes q, pargs := probeSQL(probes) args := append([]any{`["R0000"]`}, pargs...) rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...) @@ -308,16 +309,16 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) { t.Fatal(err) } defer rows.Close() - var plan []string + var steps []string for rows.Next() { var id, parent, notused int var detail string if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { t.Fatal(err) } - plan = append(plan, detail) + steps = append(steps, detail) } - joined := strings.Join(plan, " | ") + joined := strings.Join(steps, " | ") if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") { t.Fatalf("filter probe is not a primary-key range read: %s", joined) } @@ -325,3 +326,30 @@ func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) { return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video") }), "R0000") } + +// When every part is broad (above estimateCap), the smallest is still +// collected and drives — no walk over the roots — and the result is exact. +func TestSearchBroadPartsStillDriveFromTheSmallest(t *testing.T) { + defer func(old int) { estimateCapForTest(old) }(estimateCapForTest(3)) + c := newTestClient(t) + recs := map[string]map[string]string{} + for i := 0; i < 20; i++ { + ft := "video" + if i%2 == 1 { + ft = "audio" + } + recs[fmt.Sprintf("R%02d", i)] = map[string]string{"title": "Common words", "fileType": ft} + } + seed(t, c, recs) + filters := []SearchFilter{{Property: "fileType", Values: []string{"audio"}}} + plan, err := c.planCandidates(context.Background(), c.Store(), "common", filters) + if err != nil || plan.walk || len(plan.ordered) != 20 || len(plan.probes) != 1 { + t.Fatalf("want the text (20 rids) driving with one probe, got walk=%v ordered=%d probes=%d err %v", plan.walk, len(plan.ordered), len(plan.probes), err) + } + got := search(t, c, "common", filters, func(id string, f map[string]string) bool { + return textMatch("common")(id, f) && matchFilter(f, "fileType", false, "audio") + }) + if len(got) != 10 || got[0] != "R01" { + t.Fatalf("got %v", got) + } +} diff --git a/internal/storage/stats.go b/internal/storage/stats.go index fade2c0..d4fdfea 100644 --- a/internal/storage/stats.go +++ b/internal/storage/stats.go @@ -4,8 +4,9 @@ import ( "time" ) -// CountRoots returns the number of metadata roots in the root index — one -// range count over idx:roots/, never the list itself. +// CountRoots returns the number of metadata roots: a counter the write path +// keeps exact (see onChange), so it costs one read — search reports it as +// `total` on every query. func (c *Client) CountRoots() (int64, error) { s, err := c.kv() if err != nil { @@ -13,7 +14,8 @@ func (c *Client) CountRoots() (int64, error) { } ctx, cancel := opCtx() defer cancel() - return s.Count(ctx, rootsNS) + n, _, err := s.Counter(ctx, rootsCountKey) + return n, err } // keyCountTTL bounds how stale CountKeys may be. Counting every row is a full diff --git a/internal/storage/write_bench_test.go b/internal/storage/write_bench_test.go index 0455a5c..d7845bc 100644 --- a/internal/storage/write_bench_test.go +++ b/internal/storage/write_bench_test.go @@ -57,3 +57,53 @@ func BenchmarkConcurrentRecordWrites(b *testing.B) { wg.Wait() b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s") } + +// BenchmarkConcurrentRepersistUnchanged is the dominant write on a permanent +// store: a gateway re-persisting hits it already holds. The kv layer skips +// every unchanged field, so this should cost reads only. +func BenchmarkConcurrentRepersistUnchanged(b *testing.B) { + c := NewClient() + if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil { + b.Fatal(err) + } + defer c.Close() + const distinct = 2000 + rec := func(i int64) map[string]string { + m := map[string]string{ + "title": fmt.Sprintf("Some Show S01E%02d", i%40), + "fileType": "video", + fmt.Sprintf("cids/bafkbench%020d", i): "true", + } + for f := 0; len(m) < 40; f++ { + m[fmt.Sprintf("categories/%d", f)] = "true" + } + return m + } + for i := int64(0); i < distinct; i++ { + if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), rec(i)); err != nil { + b.Fatal(err) + } + } + const writers = 16 + var next int64 + b.ResetTimer() + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + i := atomic.AddInt64(&next, 1) + if i > int64(b.N) { + return + } + if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i%distinct), rec(i%distinct)); err != nil { + b.Error(err) + return + } + } + }() + } + wg.Wait() + b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s") +} diff --git a/internal/watchers/handlers.go b/internal/watchers/handlers.go index 94872f7..0b3a55c 100644 --- a/internal/watchers/handlers.go +++ b/internal/watchers/handlers.go @@ -2,29 +2,20 @@ package watchers import ( "encoding/json" - "log" "net/http" "github.com/gorilla/mux" ) -// RepublishCallback is a function type for republishing metadata events -type RepublishCallback func() (int, error) - // Handlers provides HTTP handlers for watchers management type Handlers struct { - manager *Manager - poller *Poller - republishCallback RepublishCallback + manager *Manager + poller *Poller } // NewHandlers creates new watchers handlers -func NewHandlers(manager *Manager, poller *Poller, republishCallback RepublishCallback) *Handlers { - return &Handlers{ - manager: manager, - poller: poller, - republishCallback: republishCallback, - } +func NewHandlers(manager *Manager, poller *Poller) *Handlers { + return &Handlers{manager: manager, poller: poller} } // RegisterRoutes registers all watchers-related routes @@ -215,22 +206,13 @@ func (h *Handlers) handleResetAll(w http.ResponseWriter, r *http.Request) { return } - // Republish all metadata events after reset - republishedCount := 0 - if h.republishCallback != nil { - count, err := h.republishCallback() - if err != nil { - log.Printf("[Handlers] Warning: failed to republish metadata after reset: %v", err) - } else { - republishedCount = count - } - } - + // No metadata republish: a reset rescans files; it does not change + // metadata consumers hold. A consumer that wants the whole state asks + // /api/events/meta for "0" and gets a snapshot. writeJSON(w, http.StatusOK, map[string]interface{}{ - "status": "ok", - "message": "All watchers reset", - "totalFiles": totalCount, - "republishedEvents": republishedCount, + "status": "ok", + "message": "All watchers reset", + "totalFiles": totalCount, }) }