From dc5cbdfb9b5fa19f3912485c828db1b226e9f556 Mon Sep 17 00:00:00 2001 From: Worph Date: Mon, 19 Oct 2026 20:51:00 +0000 Subject: [PATCH] replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search) --- Dockerfile | 17 +- README.md | 117 +- cmd/meta-core/main.go | 33 +- dashboard/src/pages/Dashboard.tsx | 6 +- dashboard/src/pages/Identity.tsx | 10 +- docker/entrypoint.sh | 2 +- docker/leader-election.sh | 7 +- docker/supervisord.conf | 20 +- docs/api-mediated-access.md | 60 +- docs/meta-core-architecture.md | 117 +- docs/metadata-storage-structure.md | 248 ++- docs/uuid-rooted-metadata.md | 11 +- editor/src/api/snapshotApi.ts | 2 +- editor/src/components/SnapshotPanel.tsx | 2 +- go.mod | 19 +- go.sum | 72 +- internal/api/admin_migrate.go | 131 -- internal/api/admin_sweep_test.go | 93 - internal/api/files_register_test.go | 41 +- internal/api/files_tuples_test.go | 10 +- internal/api/identity_gate_test.go | 11 +- internal/api/kv.go | 6 +- internal/api/metadata.go | 11 +- internal/api/schema_handlers.go | 23 +- internal/api/search_index.go | 228 +- internal/api/server.go | 67 +- internal/api/snapshot.go | 2 +- internal/api/sse_events.go | 159 +- internal/api/stats.go | 10 +- internal/api/storage_helper_test.go | 19 + internal/api/udl.go | 6 +- internal/api/udl_test.go | 12 +- internal/config/config.go | 18 +- internal/events/meta_publisher.go | 215 +- internal/identity/verify.go | 2 +- internal/kv/kv.go | 713 ++++++ internal/kv/kv_test.go | 314 +++ internal/kv/stream.go | 356 +++ internal/leader/election.go | 8 +- internal/schema/classify.go | 2 +- internal/schema/indexer.go | 277 +-- internal/schema/indexer_test.go | 56 + internal/snapshot/export.go | 2 +- internal/snapshot/flatten.go | 6 +- internal/snapshot/import.go | 4 +- internal/snapshot/wipe.go | 2 +- internal/storage/cid_resolution.go | 456 ++-- internal/storage/cid_retraction_test.go | 35 +- internal/storage/clear_test.go | 196 +- internal/storage/client.go | 1907 ++++++----------- internal/storage/domain_screen_migration.go | 193 -- .../storage/domain_screen_migration_test.go | 140 -- internal/storage/dual_root_migration.go | 41 - internal/storage/field_index.go | 486 ----- internal/storage/field_index_test.go | 464 ---- internal/storage/literature_echo_sweep.go | 326 --- .../storage/literature_echo_sweep_test.go | 249 --- internal/storage/schema_version.go | 99 +- internal/storage/search.go | 565 +++++ internal/storage/search_dirty.go | 245 --- internal/storage/search_dirty_test.go | 230 -- internal/storage/search_index_exclude_test.go | 116 - internal/storage/search_test.go | 327 +++ internal/storage/stats.go | 63 +- internal/storage/udl.go | 515 ++--- internal/storage/udl_admin.go | 220 +- internal/storage/udl_admin_test.go | 36 +- internal/storage/udl_export_test.go | 82 +- internal/storage/udl_test.go | 2 +- internal/storage/uuid.go | 2 +- internal/storage/uuid_test.go | 10 +- internal/storage/write_bench_test.go | 59 + internal/watcher/dispatcher.go | 20 +- internal/watcher/handlers.go | 4 +- internal/watcher/types.go | 2 +- internal/watcher/watcher.go | 14 +- internal/watchers/handlers.go | 6 +- internal/watchers/poller.go | 4 +- internal/watchers/types.go | 2 +- test/config_test.go | 15 +- 80 files changed, 4601 insertions(+), 6077 deletions(-) delete mode 100644 internal/api/admin_migrate.go delete mode 100644 internal/api/admin_sweep_test.go create mode 100644 internal/api/storage_helper_test.go create mode 100644 internal/kv/kv.go create mode 100644 internal/kv/kv_test.go create mode 100644 internal/kv/stream.go create mode 100644 internal/schema/indexer_test.go delete mode 100644 internal/storage/domain_screen_migration.go delete mode 100644 internal/storage/domain_screen_migration_test.go delete mode 100644 internal/storage/dual_root_migration.go delete mode 100644 internal/storage/field_index.go delete mode 100644 internal/storage/field_index_test.go delete mode 100644 internal/storage/literature_echo_sweep.go delete mode 100644 internal/storage/literature_echo_sweep_test.go create mode 100644 internal/storage/search.go delete mode 100644 internal/storage/search_dirty.go delete mode 100644 internal/storage/search_dirty_test.go delete mode 100644 internal/storage/search_index_exclude_test.go create mode 100644 internal/storage/search_test.go create mode 100644 internal/storage/write_bench_test.go diff --git a/Dockerfile b/Dockerfile index 14640e2..46aeed6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,20 +6,20 @@ # - Bash-based leader election script (flock on shared volume) # - meta-core Go binary (HTTP API, Mount API, Watcher) # - nginx (dashboard, WebDAV, API proxy) -# - Redis (managed by supervisord on leader) +# - SQLite metadata store, embedded in the Go binary (/meta-core/db/meta-core.db) # - rclone (single mount executor — SMB goes through rclone's :smb: backend) # - supervisord (process management - runs on leader only) # # Architecture: # - leader-election.sh handles flock, starts supervisord on leader # - Followers remain dormant, retry flock every 5s -# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher +# - Only leader runs meta-core, nginx, rclone, mount-watcher # ============================================================================= # ----------------------------------------------------------------------------- # Stage 1: Build Go binary # ----------------------------------------------------------------------------- -FROM golang:1.21-alpine AS go-builder +FROM golang:1.26-alpine AS go-builder RUN apk add --no-cache git @@ -74,7 +74,7 @@ ARG RCLONE_VERSION=1.73.5 # Install runtime dependencies. Native NFS/CIFS clients intentionally absent — # all remote mounts go through rclone now. RUN apk add --no-cache \ - redis \ + sqlite \ nginx \ nginx-mod-http-dav-ext \ supervisor \ @@ -104,7 +104,7 @@ RUN chmod +x /usr/local/bin/meta-core # Create directories RUN mkdir -p \ /meta-core/locks \ - /meta-core/db/redis \ + /meta-core/db \ /meta-core/services \ /meta-core/mounts/errors \ /files \ @@ -140,13 +140,12 @@ ENV META_CORE_PATH=/meta-core \ SERVICE_VERSION=1.0.0 \ META_CORE_HTTP_PORT=9000 \ META_CORE_HTTP_HOST=0.0.0.0 \ - REDIS_PORT=6379 \ WATCH_INTERVAL_MS=1000 # Expose ports -# 80: nginx (dashboard + API proxy + WebDAV) -# 6379: Redis (for other services) -EXPOSE 80 6379 +# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port: +# it is a file inside this container, reachable only through the HTTP API. +EXPOSE 80 # Health check HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \ diff --git a/README.md b/README.md index 16a5f7e..4a37557 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # meta-core -Standalone Go service that owns Redis, the HTTP+SSE metadata surface, the +Standalone Go service that owns the metadata store (an embedded SQLite file), +the HTTP+SSE metadata surface, the User Data Layer store and signing keystore, and a WebDAV file server for the MetaMesh dev/prod stack. @@ -10,19 +11,19 @@ meta-core runs as its own container (`metacore-app`). Other MetaMesh services (meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share, meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 — its `metamesh.core` advertise carries the full `/urls` payload) and then call its -HTTP API. `META_CORE_URL` on a client pins it instead. Redis is not published -outside the container — all metadata I/O is mediated by the HTTP/SSE API +HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded +in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API (see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)). ### Key capabilities | Feature | Description | |---------|-------------| -| **Redis-volume mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two Redis writers. It is a mutex, not discovery — nothing reads the lock. | -| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. | +| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. | +| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). | | **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. | -| **In-memory search index** | `POST /api/metadata/search` matches against an in-memory snapshot of every record, kept current by incremental refreshes (full reconcile every 30 min) — no Redis round-trip per record on the request path (`internal/api/search_index.go`). | -| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. | +| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field` over (field name, value) of every record key, plus an FTS5 trigram table — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). | +| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. | | **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. | | **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. | | **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. | @@ -35,13 +36,15 @@ outside the container — all metadata I/O is mediated by the HTTP/SSE API ### Design characteristics - **Single writer** — flock on the `/meta-core` volume; no external consensus. -- **HTTP-only externally** — Redis is never published by the compose stacks; - debug with `docker exec metacore-app redis-cli …`. +- **HTTP-only externally** — there is no database port; debug read-only with + `docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …` + (dev stacks: `/meta-core-db/meta-core.db`). - **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the - on-disk Redis layout predates the current build (alpha clean-wipe policy; + on-disk store layout predates the current build (alpha clean-wipe policy; see `docs/uuid-rooted-metadata.md`). -- **Static binary** — CGO disabled, single Go binary plus a thin Alpine - image with `redis-server`, `rclone`, nginx, and supervisord. +- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a + thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI + (for debugging). ## Architecture @@ -52,19 +55,19 @@ outside the container — all metadata I/O is mediated by the HTTP/SSE API │ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │ │ then exec supervisord) │ │ supervisord │ -│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │ │ ├─ nginx :80 (dashboard, /editor, API proxy, │ │ │ proxy_cache for /webdav, rclone GUI /rclone) │ │ ├─ rclone rcd (mount daemon, RC API on :5572) │ │ ├─ mount-watcher.sh (rclone mount lifecycle) │ │ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │ +│ ─► embedded SQLite (db/meta-core.db) │ │ ─► UDP advertise/listen 239.255.99.1:9099 │ └──────────────────────────────────────────────────────────────────────────┘ │ ┌──────────────────────┴──────────────────────┐ │ /meta-core (meta-core's own volume) │ │ locks/kv-leader.lock │ - │ db/redis/ │ + │ db/meta-core.db (+ -wal, -shm) │ │ identity/accounts/*.json │ │ mounts/{mounts.json, errors/} │ │ watchers.json │ @@ -73,7 +76,7 @@ outside the container — all metadata I/O is mediated by the HTTP/SSE API meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / … │ 1. hear the metamesh.core advertise on UDP multicast (carries /urls) - └─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never Redis, never the volume + └─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume ``` ## Quick start @@ -138,28 +141,32 @@ Environment variables (from `internal/config/config.go` unless noted): | Variable | Default | Description | |---|---|---| -| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, Redis data, identity, mounts, watchers, cache). No other service mounts it. | +| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. | | `FILES_PATH` | `/files` | Files volume root. | | `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. | | `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. | | `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. | | `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). | | `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). | -| `REDIS_PORT` | `6379` | Local Redis port. | +| `META_CORE_DB_PATH` | `/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). | +| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). | +| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. | +| `META_CORE_DB_READERS` | `8` | Reader connection pool size. | | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. | | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. | | `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). | | `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. | | `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. | | `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. | -| `META_CORE_INDEX_EXCLUDE_PREFIXES` | `categories/newznab/` | Comma-separated field prefixes left out of the in-memory search index; set empty to index everything (`internal/storage/client.go`). | | `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). | `HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`, `CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and `DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads them (leftovers of the file-based registry); setting them has no effect. There -is no `META_CORE_REDIS_PORT` / `META_CORE_SERVICE_NAME` / +is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES` +(gone with Redis and the in-memory search index; search results now include +`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` / `META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher configs created via `/api/watchers`. @@ -174,10 +181,10 @@ the in-container Go port `:9000`; from the host use ### Health, leader, URLs ```bash -GET /health # storage + role +GET /health # storage + role (`redis: bool` kept for compatibility = store open) GET /status # version, uptime, etc. GET /leader # current leader info (hostname, PID, timestamps) -GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty) +GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility) GET /neighbors # neighbours heard over UDP (beacon v2) GET /api/neighbors # same, under /api GET /api/stats # dashboard counters (see below) @@ -196,9 +203,9 @@ Every headline counter in one request, split by cost: ```jsonc { - "records": 88709, // SCARD file:__index__ — metadata roots, file or not - "redisKeys": 3115447, // DBSIZE — the flat per-field keys dominate - "redisMemory": "588.74M", + "records": 88709, // count of idx:roots/* — metadata roots, file or not + "redisKeys": 3115447, // kv row count (name kept for compatibility; cached 60 s) + "redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility) "identities": 3, // signing accounts on disk "files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep "udlUsers": 5, // accounts holding User Data Layer records @@ -207,11 +214,11 @@ Every headline counter in one request, split by cost: } ``` -The first four are O(1) commands and are live on every call. `files` and +`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and `udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record box), so they are served from a single-flight cache refreshed in the background with a 60s TTL — the walk is **never** on the request path, because the -dashboard polls and ten open tabs must not cost ten Redis walks. Prefer this +dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte` across records that share a file. @@ -266,10 +273,10 @@ PUT /api/metadata/{hashId}/property GET /api/kv/info GET /api/kv/keys -GET /api/kv/tree +GET /api/kv/tree # cursor is an opaque next-key ("0" when exhausted) GET /api/kv/search GET /api/kv/find -GET /api/kv/value +GET /api/kv/value # type is always "string" for an existing key PUT /api/kv/value DELETE /api/kv/value GET /api/kv/key/{key...} @@ -284,7 +291,8 @@ POST /api/snapshot/wipe ### SSE event streams -The HTTP-mirror of the Redis Streams. External services never read Redis +The HTTP-mirror of the store's event streams (Redis Streams until 2026-10; +IDs keep the `-` form). External services never read the store directly — they subscribe here with `Last-Event-ID` for resume. See [`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire contract (one event per stream entry, opaque IDs, heartbeats, gap @@ -318,7 +326,7 @@ GET /api/identity/aead-key Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR; writes are version-gated (stale → `409 stale_version`, or `accepted:false` per -cell in a batch). Redis model in `internal/storage/udl.go`. +cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec://` → JSON `{record,version,ts,value?}`; index sets as one key per member). ```bash GET /api/udl/record?uid&cid&key @@ -366,24 +374,21 @@ The legacy `/api/scan/trigger` and `/api/scan/status` still respond but return a deprecation pointer at the `/api/watchers/*` routes (`internal/watcher/handlers.go:32-33`). -### Admin / WebDAV +### WebDAV ```bash -POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries -POST /api/admin/migrate-domain-screen # domain film|tv → screen, workForm backfill (idempotent) -POST /api/admin/migrate-user-cids # backfill udl:idx:user::cids; until run, pre-index - # profiles export as EMPTY. Returns {fixed} -POST /api/admin/sweep-literature-echoes[?apply=true] - # strip query echoes off literature cards: anilistid on site cards, - # workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card. - # Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey} GET/PUT/DELETE /webdav/... # mounted on /files ``` +The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`, +`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were +removed with the 2026-10 SQLite switch; they had been run once on +watch.nsl.sh before the migration. + No `/metrics` endpoint is exposed; observability is via container logs and the `/health` / `/status` JSON. -## Redis-volume mutex +## DB-file mutex `docker/leader-election.sh` is a bash flock loop, not a Go-level state machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the @@ -392,8 +397,8 @@ retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree dies, the kernel releases the lock. There is no follower path inside the Go binary. -The lock is a **mutex, not service discovery** — it only stops two Redis -writers landing on one RDB/AOF. Nothing reads it, and the `kv-leader.info` +The lock is a **mutex, not service discovery** — it only stops two +writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info` file it used to publish is gone: siblings locate meta-core over UDP (see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)). `internal/leader` keeps its name for history, but only builds the `/urls` @@ -402,8 +407,8 @@ and the UDP announce share. ## Metadata storage shape (one-line summary) -Roots are opaque UUIDv7 strings; every property is a separate Redis -STRING at `file:/`; every known CID is a key-set member +Roots are opaque UUIDv7 strings; every property is a separate row +of the `kv` table at `file:/`; every known CID is a key-set member `file:/cids/` plus a reverse-index entry `cid: → ` (the canonical CID is derived by rank on read, never stored). Full details in [`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md) @@ -415,15 +420,16 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). | Package | Purpose | |---------|---------| -| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + UDP discovery + service wiring. | +| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. | | `internal/config` | Env-driven configuration, path helpers. | | `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. | | `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. | -| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, field indexes + search-index bulk reads, UDL store (`udl.go`, `udl_admin.go`), stats, one-shot migrations/sweeps. | +| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). | +| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field` + FTS (`search.go`), UDL store (`udl.go`, `udl_admin.go`), stats. | | `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). | | `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. | -| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). | -| `internal/api` | HTTP server + all handlers + SSE event endpoints + in-memory search index + `/api/stats`. | +| `internal/events` | Boot-time republish of every field into `meta:events` (`meta_publisher.go`); live events are written by `internal/kv`. | +| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. | | `internal/schema` | Live per-field schema indexer (consumes `meta:events`). | | `internal/snapshot` | Snapshot export / import / wipe. | | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. | @@ -436,14 +442,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). ``` 1. Load configuration from environment 2. Build LeaderInfoProvider (no election — the bash gate already won) -3. Connect to local Redis (retry up to 30× / 1s) -4. EnsureSchemaVersion — abort if the existing Redis layout is stale +3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db) +4. EnsureSchemaVersion — abort if the existing store layout is stale 5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false) 6. Construct API server (identity keystore migration, watcher, watchers, mounts, WebDAV) -7. Start API server + pollers + search-index refresh loop; start - MetaPublisher + schema Indexer, then warm field indexes and republish - existing metadata onto meta:events in the background +7. Start API server + pollers; start MetaPublisher + schema Indexer, then + republish existing metadata onto meta:events in the background + (5k events per transaction) 8. Wait for SIGINT/SIGTERM, shutdown in reverse order ``` @@ -452,13 +458,12 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). | Dependency | Purpose | |------------|---------| | `github.com/gorilla/mux` | HTTP router | -| `github.com/redis/go-redis/v9` | Redis client | +| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) | | `github.com/google/uuid` | UUIDv7 minting | | `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation | | `golang.org/x/net` | WebDAV support | -| `github.com/alicebob/miniredis/v2` | In-memory Redis for tests | -Runtime requirements: Go 1.21+ at build time; the container ships Redis, +Runtime requirements: Go 1.21+ at build time; the container ships rclone, nginx, and supervisord alongside the binary, plus the React dashboard and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile). diff --git a/cmd/meta-core/main.go b/cmd/meta-core/main.go index 391d1b0..a96d18d 100644 --- a/cmd/meta-core/main.go +++ b/cmd/meta-core/main.go @@ -30,36 +30,21 @@ func main() { // Create leader info provider (Go binary only runs as leader - followers loop in bash) leaderProvider := leader.NewLeaderInfoProvider(cfg) - // Connect to local Redis (supervisord starts Redis before meta-core) - // Retry connection since Redis may still be loading its dataset - redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort) - log.Printf("[meta-core] Connecting to Redis at %s", redisURL) - - storageClient := storage.NewClient("") - maxRetries := 30 - retryDelay := 1 // seconds - var err error - for i := 0; i < maxRetries; i++ { - err = storageClient.Connect(redisURL) - if err == nil { - break - } - log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err) - time.Sleep(time.Duration(retryDelay) * time.Second) + // Open the metadata store: an embedded SQLite file (internal/kv). There + // is no separate database process to wait for. + storageClient := storage.NewClient() + if err := storageClient.Connect(cfg.DBPath()); err != nil { + log.Fatalf("[meta-core] Failed to open the metadata store: %v", err) } - if err != nil { - log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err) - } - log.Println("[meta-core] Connected to Redis") // Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when - // the existing Redis store predates the UUID-rooted schema, so legacy - // data can't silently leak into the new code paths. See + // the existing store predates the UUID-rooted schema, so legacy data + // can't silently leak into the new code paths. See // docs/uuid-rooted-metadata.md. if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil { log.Fatalf("[meta-core] %v", err) } - log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion) + log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion) // Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core // as metamesh.core — carrying the /urls block services locate us by — plus @@ -101,7 +86,7 @@ func main() { return "running", []meshdisco.Resource{core, service} }, }) - if err := mesh.Start(); err != nil { + if err := mesh.Start(); err != nil { log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err) mesh = nil } diff --git a/dashboard/src/pages/Dashboard.tsx b/dashboard/src/pages/Dashboard.tsx index ac5d29d..26a1221 100644 --- a/dashboard/src/pages/Dashboard.tsx +++ b/dashboard/src/pages/Dashboard.tsx @@ -215,9 +215,9 @@ export default function Dashboard() { } />

- Redis: {health.redis ? 'Connected' : 'Disconnected'} + Metadata store: {health.redis ? 'Open' : 'Unavailable'}

) : ( diff --git a/dashboard/src/pages/Identity.tsx b/dashboard/src/pages/Identity.tsx index ca8da80..1fa66c7 100644 --- a/dashboard/src/pages/Identity.tsx +++ b/dashboard/src/pages/Identity.tsx @@ -179,7 +179,7 @@ async function fetchAccounts(): Promise { // Resolve an account's display name from the User Data Layer, or null. // // Never throws and never reports an error upward: a name is decoration on top -// of the uid, and the account list must render identically whether Redis is up +// of the uid, and the account list must render identically whether the store is up // (identity itself is file-backed, the UDL is not). The name is also *unsigned* // — meta-core stores the plaintext beside the signed record and verifies // nothing — so the uid stays on screen as the real identity, and the length cap @@ -209,7 +209,7 @@ async function fetchDisplayName(uid: string): Promise { // so per-row calls would walk the keyspace once per account. // // Never throws, for the same reason fetchDisplayName does not: the counts are -// context on top of the uid, and the list must render identically when Redis is +// context on top of the uid, and the list must render identically when the store is // down — accounts are file-backed, their data is not. // // Returns null on failure rather than an empty map. The difference is @@ -603,7 +603,7 @@ function AccountCard({ records === 1 ? 'record' : 'records' } — likes, ratings, watch progress across ${titles.toLocaleString()} ${ titles === 1 ? 'title' : 'titles' - } — are permanently deleted from Redis, along with every index that names this uid.`}{' '} + } — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '} This cannot be undone and there is no backup.
  • @@ -655,12 +655,12 @@ export default function Identity() { return; } if (gen !== reqGen.current) return; - // Render the list before resolving names, so a down Redis delays nothing. + // Render the list before resolving names, so a down store delays nothing. setAccounts(list); setLoading(false); // Names and counts both come after the list is on screen, and neither can - // fail the page: a down Redis costs decoration, not the account list. + // fail the page: a down store costs decoration, not the account list. const [settled, counts] = await Promise.all([ Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))), fetchStats(), diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index ce0904d..7605a49 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,7 +9,7 @@ set -e # Create required directories mkdir -p /meta-core/locks -mkdir -p /meta-core/db/redis +mkdir -p /meta-core/db mkdir -p /meta-core/mounts/errors mkdir -p /files mkdir -p /var/log/supervisor diff --git a/docker/leader-election.sh b/docker/leader-election.sh index 12c7135..2872325 100644 --- a/docker/leader-election.sh +++ b/docker/leader-election.sh @@ -4,9 +4,10 @@ # # The flock here is a MUTEX, not service discovery. It is the only thing # stopping two containers that mount the same /meta-core volume from putting -# two Redis writers on one RDB/AOF. Sibling services no longer read anything -# from this directory — they locate meta-core over UDP (meta-discovery v1, see -# docs/project-architecture/service-discovery.md), so the kv-leader.info file +# two writers on one database file (/meta-core/db/meta-core.db). Sibling +# services no longer read anything from this directory — they locate meta-core +# over UDP (beacon v2, see docs/project-architecture/beacon-v2.md), so the +# kv-leader.info file # this script used to publish is gone. Keep the lock; it costs nothing and the # corruption it prevents is unrecoverable. # diff --git a/docker/supervisord.conf b/docker/supervisord.conf index 62cd1aa..5aa8a5b 100644 --- a/docker/supervisord.conf +++ b/docker/supervisord.conf @@ -16,26 +16,13 @@ serverurl=unix:///var/run/supervisor.sock supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface # ============================================================================= -# Redis: Key-value store (managed by supervisord, started before meta-core) -# Previously spawned by Go code, now managed by bash-based leader election -# ============================================================================= -[program:redis] -command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$ -priority=1 -autostart=true -autorestart=true -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -startsecs=2 - -# ============================================================================= -# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis) +# meta-core: HTTP API, Mount API, File Watcher, metadata store (embedded SQLite) # ============================================================================= [program:meta-core] command=/usr/local/bin/meta-core priority=10 +; Room to finish in-flight writes and checkpoint the WAL on shutdown. +stopwaitsecs=60 autostart=true autorestart=true stdout_logfile=/dev/stdout @@ -50,7 +37,6 @@ environment= SERVICE_VERSION="%(ENV_SERVICE_VERSION)s", META_CORE_HTTP_PORT="%(ENV_META_CORE_HTTP_PORT)s", META_CORE_HTTP_HOST="%(ENV_META_CORE_HTTP_HOST)s", - REDIS_PORT="%(ENV_REDIS_PORT)s", WATCH_INTERVAL_MS="%(ENV_WATCH_INTERVAL_MS)s", API_PORT="80", BASE_URL="%(ENV_BASE_URL)s" diff --git a/docs/api-mediated-access.md b/docs/api-mediated-access.md index 7fff346..dc05775 100644 --- a/docs/api-mediated-access.md +++ b/docs/api-mediated-access.md @@ -9,6 +9,16 @@ on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes meta-core is the single writer to the metadata keyspace — an assumption that's now actually enforced rather than aspirational. +> **2026-10: meta-core no longer uses Redis at all.** Its store is now an +> embedded SQLite file (`/meta-core/db/meta-core.db`, see +> [metadata-storage-structure.md](metadata-storage-structure.md)) and the +> `file:events` / `meta:events` streams are rows of that store. The lockdown +> below is now structural — there is no database port anywhere — and the +> SSE wire contract was kept byte-for-byte, so no consumer changed. The +> rest of this doc is the design record of the Redis-era lockdown; the +> sections that describe *current* behaviour (SSE retention, reconnect, +> error handling) have been updated in place. + See "Implementation outcome" at the bottom for the delta between this doc's plan and what shipped (notably the write-path CID resolution and the dual-root migration sweep, which weren't in the original plan but turned @@ -85,7 +95,7 @@ we don't fully control. ``` **The single-writer rule.** meta-core has exclusive write authority over -the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No +the `file:*`, `cid:*`, `file:__index__` (now `idx:*`), and `meta-core:*` keyspaces. No other service can write — not because of an honoured convention, but because they can't reach Redis at all. @@ -243,6 +253,25 @@ After PR A, the cursor contract is **"durable across meta-core restart and bounded by `MAXLEN ~ N`"** — matching what consumers already assume when they persist a Last-Event-ID. +**Current backing store (since 2026-10).** The streams are rows of +meta-core's SQLite kv table — `ev:/-` → JSON of the +entry's fields, plus a high-water mark `evhwm:` that seeds the next +ID (so a `ClearStream` reset never reissues an ID an old cursor treats as +seen). The contract above holds unchanged: + +- Entries persist across restart (they are in the database file). +- Retention is ~100k entries per stream, trimmed every 10k appends. +- IDs keep the Redis `-` form, so cursors persisted by meta-sort / + meta-share / meta-stremio before the switch still compare. +- `meta:events` entries (`{type: set|del, key, ts}`) are appended by the + storage engine **in the same transaction** as the `file:` key change — + `del` only when the key existed, `set` even for an unchanged value. The + Redis keyspace-notification hop (PSUBSCRIBE in `MetaPublisher`), which + could drop events when it lagged, is gone. +- The boot-time republish of every field into `meta:events` is kept + (meta-fuse rebuilds its VFS only by replaying from `0-0`), batched 5k + events per transaction. + ### Heartbeats Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds @@ -254,10 +283,13 @@ per the SSE spec. Per the SSE spec the client (or `EventSource`) auto-reconnects after a network error, sending `Last-Event-ID`. The server resumes from the next -entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events `. +entry after that ID (a range read of the stream's rows; under Redis this +was `XREAD … STREAMS file:events `). With no `Last-Event-ID` the stream +starts at the head, resolved once at connect; `0` / `0-0` go through the +gap path and replay from the beginning. -**Gap handling.** If the requested ID has been trimmed out of the Redis -Stream by retention, the server emits one synthetic event before resuming +**Gap handling.** If the requested ID is older than the oldest retained +entry, the server emits one synthetic event before resuming from the oldest available entry: ``` @@ -271,9 +303,10 @@ react. Consumers that don't care ignore it. ### Error handling -- **Redis unreachable on connect**: 503 with `Retry-After: 5`. -- **Redis fails mid-stream**: close the SSE connection. Client reconnects - with `Last-Event-ID`; the cycle resumes when Redis recovers. +- **Store unavailable on connect**: 503 with `Retry-After: 5`. +- **Store read fails mid-stream**: close the SSE connection. Client + reconnects with `Last-Event-ID`; the cycle resumes when meta-core is back + (a meta-core restart is the common case — SSE clients just reconnect). - **Client disconnects**: handler goroutine exits cleanly; no per-client state on meta-core. @@ -307,7 +340,8 @@ inside-only until a concrete external consumer appears. ### Implementation footprint The SSE handler itself is ~80 lines of Go in meta-core: one handler per -stream, each running an `XREAD … BLOCK 30s` loop and writing +stream, each running a read-after-cursor loop (an `XREAD … BLOCK 30s` loop +in the Redis era) and writing SSE-formatted lines to the response. Go's `net/http` natively supports streaming responses with `http.Flusher.Flush()`. Standard SSE clients exist for every language we use: Node `eventsource`, Rust @@ -562,11 +596,11 @@ metacore-app produces a connection refused. The lock is real. | Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason | | Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit | | Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint | -| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact | +| Backing store for streams? | Redis Streams at the time; SQLite kv rows since 2026-10 | SSE wraps it; the 2026-10 swap proved it — no consumer changed | | Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it | | Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha | | **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. | -| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`-`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. | +| **SSE event-id contract?** | **Opaque to consumers** | It is `-` (the Redis Stream entry form, kept after the 2026-10 SQLite switch); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. | | **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. | | **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. | | **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. | @@ -591,7 +625,7 @@ Stable `error` slugs the client should switch on: | `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. | | `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. | | `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. | -| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. | +| `storage_unavailable` | 503 | yes | Store transient (Redis at the time; the embedded SQLite store since 2026-10). Caller should retry with backoff. | | `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. | **Implemented.** Every 4xx/5xx response across `internal/api`, @@ -709,7 +743,9 @@ contract violation." One-shot fixer for already-stranded entries: walks every UUID's `midhash256` field to build the orphan index, merges each stranded `file:/*` into the matching UUID via `MergeMetadataFlat`, - deletes the stranded root, and re-points the alias. + deletes the stranded root, and re-points the alias. *Removed in + 2026-10* with the SQLite switch, together with the other one-shot Redis + migrations; it had been run on watch.nsl.sh before the migration. Verified on the dev stack: 48 stranded entries were reunited; the editor now shows every file with its full plugin metadata under one diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index b97f521..b74fe58 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -3,14 +3,15 @@ ## Document Purpose This document describes the architectural design of **meta-core**, the -standalone container that owns Redis, the leader-info file, the HTTP+SSE +standalone container that owns the metadata store (an embedded SQLite +file; Redis until 2026-10), the leader-info file, the HTTP+SSE metadata surface, and a WebDAV file server. It covers the leader gate, storage data model, API design, the SSE event surface, and the integration contract with other MetaMesh services. -For the on-disk Redis layout, see +For the on-disk key layout, see [metadata-storage-structure.md](metadata-storage-structure.md). For the -external-access design (Redis lockdown, SSE wire contract, auth rules), +external-access design (store lockdown, SSE wire contract, auth rules), see [api-mediated-access.md](api-mediated-access.md). For the UUID-rooted storage rationale, see [uuid-rooted-metadata.md](uuid-rooted-metadata.md). @@ -31,14 +32,15 @@ view of file metadata: meta-core sits below all of these as the single owner of: -1. **Redis** — the metadata store and event source. +1. **The metadata store** — an embedded SQLite file + (`/meta-core/db/meta-core.db`) holding metadata and the event streams. 2. **The leader-info file** — the one place services look to discover meta-core's URLs. 3. **The HTTP+SSE surface** — the only way external services touch the data plane. 4. **A WebDAV server** — file access for plugins and other services. ### Core design principle -> External services never speak Redis. Every read, write, and event flows +> External services never touch the store. Every read, write, and event flows > through meta-core's HTTP+SSE surface. This is the contract that PR D of [api-mediated-access.md](api-mediated-access.md) @@ -53,8 +55,8 @@ landed and that the rest of this document assumes. meta-core runs as a dedicated container (`metacore-app`). The dev compose runs a single instance; the legacy multi-instance topology (sibling followers) was deprecated. The flock primitive is retained as the -correctness anchor — it guarantees that at most one process owns Redis and -the writable state under `/meta-core`. +correctness anchor — it guarantees that at most one process writes the +database file and the writable state under `/meta-core`. ``` ┌────────────────────────────────────────────────────────────────────────┐ @@ -62,17 +64,17 @@ the writable state under `/meta-core`. │ │ │ supervisord │ │ ├─ leader-election.sh (flock → write kv-leader.info → exec self) │ -│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │ │ ├─ nginx (TLS termination, proxy_cache for /webdav) │ │ ├─ rclone rcd (mount daemon) │ │ ├─ mount-watcher.sh (rclone mount lifecycle) │ │ └─ meta-core (Go) ──► HTTP + SSE + WebDAV on :9000 │ +│ + embedded SQLite (db/meta-core.db) │ └────────────────────────────────────────────────────────────────────────┘ │ ┌────────────────────────────┴────────────────────────────┐ │ /meta-core (shared volume) │ │ locks/kv-leader.{lock,info} │ - │ db/redis/ │ + │ db/meta-core.db (+ -wal, -shm) │ │ services/*.json │ │ mounts/{mounts.json, errors/} │ │ watchers.json │ @@ -152,7 +154,7 @@ Container startup ▼ ┌──────────────────────────────┐ │ supervisord starts: │ -│ redis → nginx → rclone → │ +│ nginx → rclone → │ │ meta-core Go binary │ └──────────────────────────────┘ ``` @@ -167,10 +169,11 @@ gate wins. Therefore: local hostname + IP + config and returns it on demand. There are no `OnBecomeLeader` / `OnBecomeFollower` / `OnLeaderLost` callbacks; those states do not exist inside the Go process. -- `LeaderLockInfo.RedisUrl` is intentionally empty in the response from - `/urls` — see `internal/leader/election.go:82`. PR D of - api-mediated-access.md removed direct Redis exposure; consumers route - metadata I/O through meta-core's HTTP API and SSE event streams. +- `LeaderLockInfo.RedisUrl` is always empty in the response from + `/urls` (kept for wire compatibility). PR D of + api-mediated-access.md removed direct Redis exposure, and since 2026-10 + there is no Redis at all; consumers route metadata I/O through + meta-core's HTTP API and SSE event streams. ### Lock-info file @@ -192,18 +195,18 @@ Other services read this file, then call `GET /urls` for full discovery `metacore-app` is provisioned, its bash loop will acquire on next tick. The dev compose runs a single instance; failover is "restart the container." -- **Redis crash inside the container** → supervisord restarts redis. The - Go binary's connect loop retries up to 30 times (1s delay) at startup; - at steady state, individual operations fail and clients retry. +- **The store is in-process** — there is no separate database process to + crash. A meta-core crash is a process restart; SQLite's WAL recovers the + last committed transaction on reopen. --- ## Schema-Version Gate Before serving any traffic, `cmd/meta-core/main.go:54-61` calls -`storage.EnsureSchemaVersion`. The sentinel lives in Redis at +`storage.EnsureSchemaVersion`. The sentinel lives in the store at `meta-core:schema-version` (`internal/storage/schema_version.go`). The -gate refuses to boot when the existing Redis layout predates the +gate refuses to boot when the existing store layout predates the UUID-rooted schema, so legacy `file:midhash256:*` data can't leak into the current code paths. In alpha there is no automated migration — operators wipe with `pnpm run clean:all` and restart. @@ -216,20 +219,20 @@ operators wipe with `pnpm run clean:all` and restart. ``` ┌────────────────────────────────────────────────────────────────────┐ -│ METADATA (Redis) │ +│ METADATA (embedded SQLite, kv table) │ │ │ │ Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars) │ │ Storage: flat string keys │ │ file:/ → string value │ │ file:/cids/ → "true" (key-set) │ -│ file:/duplicates → SET of paths │ +│ dup:/ → "" (one per path) │ │ │ │ No stored canonical_cid — derived by rank on read. │ │ │ -│ Aliases (reverse index, one Redis STRING each): │ +│ Aliases (reverse index, one key each): │ │ cid: → │ │ │ -│ Index: file:__index__ → SET of all UUIDs │ +│ Index: idx:roots/ → "" (one per root) │ │ │ │ Mintable without a local file (remote / deleted entries OK). │ │ Resolvable by any registered CID in O(1). │ @@ -317,7 +320,7 @@ Editor / KV / schema / snapshot POST /api/snapshot/import POST /api/snapshot/wipe -SSE event streams (HTTP-mirror of Redis Streams) +SSE event streams (HTTP-mirror of the store's event streams) GET /api/events/files GET /api/events/meta @@ -337,11 +340,10 @@ Watchers POST /api/watchers/{scan-all,reset-all} (deprecated: /api/scan/trigger, /api/scan/status — see watcher/handlers.go:32) -Admin - POST /api/admin/migrate-dual-roots - POST /api/admin/sweep-literature-echoes (dry-run unless ?apply=true; - strips query echoes off domain=literature fileType=card records — - see storage/literature_echo_sweep.go) +(The one-shot Redis-era admin migrations — `/api/admin/migrate-dual-roots`, +`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes` — +were removed with the 2026-10 SQLite switch; they had been run once on +watch.nsl.sh beforehand.) WebDAV /webdav/... (GET, PUT, DELETE, MKCOL, COPY, MOVE, PROPFIND) @@ -369,12 +371,16 @@ See `docs/api-mediated-access.md` "Auth" for the rationale. ### SSE event streams — wire contract `/api/events/files` and `/api/events/meta` are SSE mirrors of the -`file:events` and `meta:events` Redis Streams. The contract +`file:events` and `meta:events` streams, stored as `ev:/…` rows of +the kv table (`internal/kv/stream.go`). The contract (`internal/api/sse_events.go`): -- One SSE event per Redis Stream entry. -- `id:` is the opaque `-` Redis Stream entry ID — clients echo - it back via `Last-Event-ID` on reconnect. +- One SSE event per stream entry. +- `id:` is the opaque `-` entry ID (the Redis Stream form, kept so + cursors persisted before the 2026-10 switch still compare) — clients echo + it back via `Last-Event-ID` on reconnect. Without `Last-Event-ID` the + stream starts at the head (resolved once at connect); `0` / `0-0` replay + from the beginning. - `event:` is the `type` field of the underlying entry. - `data:` is the JSON of the rest of the entry. - Heartbeats: SSE comment `:keep-alive\n\n` every 30s of silence. @@ -383,9 +389,9 @@ See `docs/api-mediated-access.md` "Auth" for the rationale. `event: gap` payload before resuming from the oldest available entry. External services use SSE with `Last-Event-ID` as the resume primitive. -A Redis consumer-group (XREADGROUP / XACK) exists *internally* — the -`MetaPublisher` uses one to drain keyspace notifications — but it is not -the external contract. +`meta:events` entries are appended by the storage engine in the same +transaction as the change they describe; the Redis-era keyspace-notification +hop through `MetaPublisher` is gone. ### WebDAV URLs @@ -415,7 +421,7 @@ WebDAV caching is handled by nginx's `proxy_cache` (see | Single binary | Static compilation | | Low resource | ~10-20 MB resident | | HTTP server | stdlib `net/http` + gorilla/mux | -| Redis client | go-redis | +| Metadata store | embedded SQLite via `modernc.org/sqlite` (pure Go, CGO off) | | WebDAV | `golang.org/x/net/webdav` | ### Repo layout @@ -428,12 +434,13 @@ packages/meta-core/ │ ├── cid/ CID parsing + algorithm ranking │ ├── config/ env-driven configuration │ ├── discovery/ service registry + dead-service cleaner -│ ├── events/ meta:events stream publisher +│ ├── events/ boot-time meta:events republish +│ ├── kv/ SQLite kv engine + event streams │ ├── leader/ LeaderInfoProvider (no election logic) │ ├── mounts/ rclone mount manager + handlers + stats │ ├── schema/ live per-field schema indexer │ ├── snapshot/ snapshot export / import / wipe -│ ├── storage/ Redis wrapper + UUID + CID reverse index + schema gate +│ ├── storage/ flat-key records + search + UUID + CID reverse index + schema gate │ ├── watcher/ scanner + dispatcher + midhash + state │ ├── watchers/ polling-based watcher configs │ └── webdav/ /webdav/* handler @@ -454,8 +461,8 @@ packages/meta-core/ ``` 1. Load configuration from environment 2. Build LeaderInfoProvider (no election; bash gate already won) -3. Connect to local Redis (retry up to 30× / 1s) -4. EnsureSchemaVersion (abort if Redis layout is stale) +3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db) +4. EnsureSchemaVersion (abort if the store layout is stale) 5. Start service discovery + dead-service cleaner 6. NewServer + Server.Start (boots watcher, mounts, WebDAV, API) 7. On first connection: start MetaPublisher + schema Indexer, @@ -466,9 +473,9 @@ packages/meta-core/ ### Process management -Redis is not spawned by the Go binary; supervisord starts it before -meta-core (`docker/supervisord.conf`). The Go binary connects to -`localhost:6379` and retries until Redis is reachable. +There is no database process: the store is embedded in the Go binary +(`docker/supervisord.conf` runs meta-core, nginx and rclone). Until +2026-10 supervisord also ran `redis-server` on `localhost:6379`. --- @@ -476,15 +483,16 @@ meta-core (`docker/supervisord.conf`). The Go binary connects to ### Container restart -Supervisord restarts crashed processes (meta-core, redis, nginx, rclone) +Supervisord restarts crashed processes (meta-core, nginx, rclone) individually. The bash flock is held by supervisord's process tree, so internal crashes don't trigger re-election. -### Redis crash +### Store durability -Supervisord restarts redis. The Go binary's connect retry handles the -window. Persistence is RDB + AOF on the `/meta-core` volume, so no data is -lost across restarts. +The SQLite file runs in WAL mode with `synchronous=NORMAL`: every +committed transaction survives a meta-core process crash. A record write — fields, root index, CID aliases, +FTS row and its events — is one transaction, so a crash never leaves a +half-written record. ### Container crash @@ -517,7 +525,10 @@ via `/health`; metadata I/O and SSE event streams stop. | `SERVICE_VERSION` | `1.0.0` | Reported in `/status`. | | `API_PORT` | `8180` | External port baked into `baseUrl`. | | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). | -| `REDIS_PORT` | `6379` | Local Redis port. | +| `META_CORE_DB_PATH` | `/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` (named volume — SQLite WAL is unsafe on the 9p `/meta-core` bind of Windows dev boxes). | +| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection. | +| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. | +| `META_CORE_DB_READERS` | `8` | Reader connection pool size. | | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. | | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind address. | | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. | @@ -536,7 +547,7 @@ via `/health`; metadata I/O and SSE event streams stop. |---|---| | `locks/kv-leader.lock` | Flock target. | | `locks/kv-leader.info` | Leader API URL (plaintext). | -| `db/redis/{dump.rdb, appendonly.aof}` | Redis persistence. | +| `db/meta-core.db` (+ `-wal`, `-shm`) | Metadata store (SQLite). Replaced `db/redis/` in 2026-10. | | `services/*.json` | Service registry. | | `mounts/mounts.json` | rclone mount configuration. | | `mounts/errors/{id}.error` | Mount error logs. | @@ -552,5 +563,5 @@ and file access for MetaMesh into one container with a typed HTTP+SSE surface. The bash flock gate guarantees single-leader semantics without an external consensus system. UUID-rooted storage with CID aliasing treats all content hashes symmetrically and makes any-CID resolution -O(1). API-mediated access ensures no other service ever needs to speak -Redis. +a single lookup. API-mediated access ensures no other service ever touches +the store. diff --git a/docs/metadata-storage-structure.md b/docs/metadata-storage-structure.md index e02a014..6af5304 100644 --- a/docs/metadata-storage-structure.md +++ b/docs/metadata-storage-structure.md @@ -1,6 +1,6 @@ -# Redis Metadata Storage Structure +# Metadata Storage Structure -How meta-core lays out file metadata in Redis. This document covers the +How meta-core lays out file metadata in its store. This document covers the on-disk shape (UUID-rooted flat string keys, CID reverse index, per-root sets) and the schema-version gate. It does **not** enumerate field semantics — that's the job of the repo-root @@ -12,40 +12,80 @@ reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md). For the external-access contract (HTTP-only, SSE for events), see [`api-mediated-access.md`](api-mediated-access.md). +> **Storage engine (since 2026-10): an embedded SQLite file, not Redis.** +> Until 2026-10 these keys lived in a Redis server spawned inside the +> container. They now live in one string→string table of an embedded SQLite +> database (pure-Go driver `modernc.org/sqlite`, CGO off). The **key names +> did not change** — only the Redis *types* that some bookkeeping used (SETs, +> HASHes, Streams) were flattened into plain keys, listed below. Holding +> everything in RAM had cost ~6.6 GB on watch.nsl.sh's 169k records (Redis +> plus the API's in-memory search copy) and caused repeated OOM kills. + ## Quick summary +- **One table**: `kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`, + schema-free — a new metadata field is just a new key, never a migration. + File: `/meta-core/db/meta-core.db` (override `META_CORE_DB_PATH`), WAL + mode, `synchronous=NORMAL`. - **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout), not content hashes. Examples in this doc use the placeholder `01JKR8XW5T4QMV7AHNJ2DEFGHK`. -- **Properties are flat Redis STRING keys**, one per leaf: - `file:/` → string value. *Not* Redis Hashes. +- **Properties are flat string keys**, one per leaf: + `file:/` → string value. A record is exactly the + key range `[file:/, file:0)` — there is no per-record field + list. - **CIDs are reverse-index aliases**: `cid:` → `` - (a Redis STRING per known CID; no algorithm prefix — the multicodec + (one key per known CID; no algorithm prefix — the multicodec already carries it). -- **Per-root sets**: `file:/cids/` (a *key-set*: one STRING - `"true"` per known CID, midhash included), `file:/duplicates` - (alternate paths with the same content), plus `file:__index__` (every UUID). +- **Per-root sets**: `file:/cids/` (a *key-set*: one key + `"true"` per known CID, midhash included), `dup:/` (alternate + paths with the same content), plus `idx:roots/` (every UUID). There is **no stored `canonical_cid`** — it is derived by rank on read. - **A schema-version sentinel** (`meta-core:schema-version`) refuses to boot against legacy data. -Implementation lives in `internal/storage/client.go`, -`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and -`internal/storage/schema_version.go`. +Implementation lives in `internal/kv/kv.go` (engine), +`internal/kv/stream.go` (event streams), `internal/storage/client.go`, +`internal/storage/search.go`, `internal/storage/cid_resolution.go`, +`internal/storage/uuid.go`, and `internal/storage/schema_version.go`. ## Key structure +Everything is one string→string table. The full layout: + +| Key | Value | Notes | +|---|---|---| +| `file:/` | field value | a record property — the **only** thing under `file:` | +| `cid:` | `` | reverse index | +| `meta-core:schema-version` | `"2"` | schema gate | +| `idx:roots/` | `""` | set of every root (was the Redis SET `file:__index__`) | +| `idx:fts/` | FTS rowid | the record's row in the FTS table | +| `dup:/` | `""` | a duplicate path (was the SET `file:/duplicates`) | +| `udl:rec://` | JSON `{record,version,ts,value?}` | User Data Layer entry (was a HASH) | +| `udl:idx:…\x1f` | `""` | UDL index sets, one key per member | +| `ev:/-` | JSON of the entry's fields | `meta:events` / `file:events` entries (were Redis Streams) | +| `evhwm:` | last issued ID | stream high-water mark | +| `btmagnet:*`, `searchcov:*`, `nzb:*` | — | written by other services via `/api/kv/value`; unchanged | + +Keeping bookkeeping **out of `file:`** is load-bearing: the `file:` range is +exactly the records, so the field index covers only real fields and every +change under `file:` is a real field change worth publishing on +`meta:events`. The per-record `file:/__fields__` SET and the +`file:__dirty__` set of the Redis era no longer exist. Roots are split from +the field at the **first** `/`, so CID-rooted and UUID-rooted records both +parse. + ### Roots ``` -file:/ → Redis STRING (always) +file:/ → string (always) ``` UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`): UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The -time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted -in a given window, which restored some of the debug-ability lost when -midhash256 stopped being the root. +time-sortable prefix means roots minted in a given window sort together +(`k >= 'file:01JKR8' AND k < 'file:01JKR9'`), which restored some of the +debug-ability lost when midhash256 stopped being the root. Properties use `/` as the path separator. Nested objects in source code are flattened to flat keys on write and reconstructed on read. Examples: @@ -62,7 +102,7 @@ file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih... ### Reverse-index aliases (CIDs) -Every CID known for a file is registered as its own Redis STRING, keyed by +Every CID known for a file is registered as its own key, keyed by the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash function in the multicodec, so naming it again in the key would be redundant (and would let the same digest be indexed under two spellings): @@ -78,13 +118,12 @@ cid:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK cid:bagacbabaec7v3... → 01JKR8XW5T4QMV7AHNJ2DEFGHK ``` -The reverse index makes `GET /api/meta/` an O(1) lookup for any +The reverse index makes `GET /api/meta/` a primary-key lookup for any registered CID, independent of which hash algorithm — there is no primary. -The auto-alias hook (`maybeAddAliasFromFieldLocked` in -`internal/storage/cid_resolution.go`, called from `SetProperty`, -`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any -field whose name starts with `cids/`, so writers emit key-set members -without knowing the reverse index exists. +The record write path (`writeFieldsTx` in `internal/storage/client.go`, used +by `SetProperty`, `SetMetadataFlat` and `MergeMetadataFlat`) registers the +alias for any field whose name starts with `cids/`, in the same transaction, +so writers emit key-set members without knowing the reverse index exists. > ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or > `midhash256` *named* field written to a root is stored verbatim and is @@ -95,12 +134,13 @@ without knowing the reverse index exists. ### Per-root sets ``` -file:/cids/ → STRING "true" — one key per known CID (a key-set, not a Redis SET) -file:/duplicates → Redis SET of alternate paths with same content +file:/cids/ → "true" — one key per known CID (a key-set) +dup:/ → "" — one key per alternate path with same content ``` -`cids` is a **key-set**: one flat `file:/cids/` STRING per member, -which keeps it conflict-free to merge across peers (a SET would not be). +`cids` is a **key-set**: one flat `file:/cids/` key per member, +which keeps it conflict-free to merge across peers (a single set-valued +field would not be). The record's own address (the midhash) is just the `0x1000` member — it has no separate field. @@ -114,61 +154,88 @@ was removed with the flat `cid_*` keys. ### Global index ``` -file:__index__ → Redis SET of all UUIDs +idx:roots/ → "" (one key per root; was the Redis SET file:__index__) ``` -`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate -every file without scanning the keyspace. +`internal/storage/client.go::GetAllHashIDs` range-reads `idx:roots/` to +enumerate every file without scanning the records. -```bash -SMEMBERS file:__index__ # all UUIDs -SCARD file:__index__ # total file count -SISMEMBER file:__index__ -``` +### Search structures (derived) + +Two derived structures are maintained on top of `kv`, in the same +transaction as every record write. Both are rebuildable from the kv rows and +never authoritative (`internal/storage/search.go`): + +- **`kv_field`** — a partial expression index + `ON kv(lower(substr(k,instr(k,'/')+1)), lower(v)) WHERE k>='file:' AND k<'file;' AND instr(k,'/')>0`. + It answers "which records have field F with a value like V" for **any** + field, including ones invented tomorrow — the index is over the key itself. +- **`fts`** — an FTS5 trigram table `fts(rid UNINDEXED, hay)` holding one row + per record: its search haystack plus its rid (`idx:fts/` points at the + row). Trigrams make every 3+ character substring indexable. + +`POST /api/metadata/search` uses SQL to build a superset of candidates, then +verifies each with the unchanged Go matchers (`recordMatchesFilter`, +`matchAllTokens`), so semantics are identical to the old in-memory index. +Results still come in hashID order and are fresh immediately (no refresh +lag). This replaced the API's in-memory `searchIndex` (60 s refresh, ~30 KB +RAM per record). ### Schema-version sentinel ``` -meta-core:schema-version → STRING (currently "2") +meta-core:schema-version → string (currently "2") ``` Set at first boot, checked on every subsequent boot (`internal/storage/schema_version.go`). If the sentinel is missing but -`file:__index__` is non-empty, the gate refuses to boot (legacy data). +the root index is non-empty, the gate refuses to boot (legacy data). If the sentinel is present but doesn't match `SchemaVersion`, the gate also refuses to boot. Alpha-stage policy: operators wipe and restart; there is no automated migration. -## Why flat STRING keys? +## Why flat string keys? -The store does not use Redis Hashes. The reasons are: +The store keeps one flat key per leaf rather than a document or a +per-record row. The reasons are: -1. **Field-level keyspace notifications.** Redis publishes - `__keyspace@0__:` on every SET/DEL of a STRING key. With Hashes - you'd only get one notification on the hash itself. meta-core's - `MetaPublisher` (`internal/events/meta_publisher.go`) consumes these - notifications and publishes structured events to the `meta:events` - Redis Stream. -2. **Selective updates without read-modify-write.** A plugin writing +1. **Schema-free.** A plugin inventing a field writes a new key; nothing + declares it, and the table never needs a migration. The field index + covers it automatically. +2. **Conflict-free merging across peers.** Key-sets (`cids/`, and the + flat fields in general) merge by union of keys — no read-modify-write of + a shared value, nothing to reconcile. +3. **Selective updates without read-modify-write.** A plugin writing `tmdb/poster` doesn't have to read the rest of the document. -3. **Compatibility with the CID alias hooks.** Field writes can trigger - alias-maintenance work in `SetProperty` without the plugin caring. +4. **Compatibility with the CID alias hooks.** Field writes can trigger + alias-maintenance work in the write path without the plugin caring. + +(Under Redis there was a fourth reason — field-level keyspace notifications +fed `meta:events`. That is obsolete: events are now written by the storage +engine itself, see [Event streams](#event-streams).) ## Reads and writes +All writes go through **one writer goroutine** with group commit: concurrent +writes share one SQLite transaction, each under its own SAVEPOINT so a +failure rolls back alone. A record write — fields + root index + CID aliases ++ FTS row + events — is atomic. (Measured ~835 records/s with 16 concurrent +writers of 40-field records.) Reads use a separate connection pool and see +the last committed state. + The Go API in `internal/storage/client.go`: | Method | Behaviour | |---|---| -| `GetMetadataFlat(uuid)` | `SCAN` for `file:/*`, `MGET` the values, return `map[string]string`. | -| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/` member. | +| `GetMetadataFlat(uuid)` | Range-read `[file:/, file:0)`, return `map[string]string`. | +| `SetMetadataFlat(uuid, m)` | Put each field; add `idx:roots/`; auto-register an alias for every `cids/` member; refresh the FTS row. One transaction. | | `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. | -| `GetProperty(uuid, field)` | `GET file:/`. | -| `SetProperty(uuid, field, value)` | `SET file:/`; auto-register alias if the field is a CID. | -| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:/*`, unmap every reverse-index entry. | +| `GetProperty(uuid, field)` | Get `file:/`. | +| `SetProperty(uuid, field, value)` | Put `file:/`; auto-register alias if the field is a CID. | +| `DeleteRoot(uuid)` | Delete the `file:/` range, its duplicates, search row and index membership; unmap every reverse-index entry. | | `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. | | `AddAlias(uuid, "")` | Set `cid:` → `` and the key-set member `file:/cids/` = `"true"`. Nothing to reconcile — canonical is derived on read. | -| `ResolveRoot("")` | `GET cid:` → uuid (returns the input unchanged if it is already a UUID root). | +| `ResolveRoot("")` | Get `cid:` → uuid (returns the input unchanged if it is already a UUID root). | | `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. | ## Reconstruction (flat → nested) @@ -196,7 +263,7 @@ Rules: ## Plugin contribution flow -Plugins don't talk to Redis. They write metadata via meta-core's HTTP +Plugins don't talk to the store directly. They write metadata via meta-core's HTTP API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin writes under a prefix: @@ -215,12 +282,24 @@ deliberately does not duplicate that registry — it would drift. ## Event streams -Two Redis Streams are produced inside meta-core: +Two event streams are kept inside the store, as rows +`ev:/-` (zero-padded so key order is ID order) plus a +high-water mark `evhwm:` (`internal/kv/stream.go`): | Stream | Producer | Contents | |---|---|---| | `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. | -| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. | +| `meta:events` | the storage engine (`internal/kv`) | `{type: set\|del, key, ts}` for every Put/Delete of a `file:` key, appended **in the same transaction** as the change. | + +`del` is emitted only when the key existed; `set` is emitted even for an +unchanged value (Redis semantics, kept). Retention is ~100k entries, +trimmed every 10k appends. At boot `MetaPublisher` +(`internal/events/meta_publisher.go`) still republishes every field into +`meta:events` (meta-fuse rebuilds its VFS only by replaying from `0-0`), +batched 5k events per transaction. + +The Redis-era path — keyspace notifications drained by `MetaPublisher` +through a PSUBSCRIBE / consumer-group hop — is gone (2026-10). External consumers do **not** read these streams directly. They subscribe via SSE: @@ -228,51 +307,50 @@ via SSE: - `GET /api/events/files` — mirror of `file:events` - `GET /api/events/meta` — mirror of `meta:events` -The SSE wire contract uses `Last-Event-ID` for resume (the opaque -`-` Redis Stream entry ID). Heartbeats are SSE comments +The SSE wire contract uses `Last-Event-ID` for resume. IDs keep the Redis +`-` form so cursors persisted by meta-sort / meta-share / +meta-stremio across the switch still compare. Heartbeats are SSE comments (`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past a client's cursor, the handler emits one `event: gap` payload before resuming. See `internal/api/sse_events.go` and [`api-mediated-access.md`](api-mediated-access.md) for the full contract. -Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK) -inside `MetaPublisher` to drain keyspace notifications, but this is not -the external surface — external services use SSE. - ## Debugging -```bash -# Roots -docker exec metacore-app redis-cli SMEMBERS 'file:__index__' -docker exec metacore-app redis-cli SCARD 'file:__index__' -docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__' +The database path is `/meta-core/db/meta-core.db` in the image default and +on the store app; dev stacks use `/meta-core-db/meta-core.db` (a named +volume — the `/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL +is unsafe). Always open it **read-only**; writes must go through the API. -# Full document for one root (flat scan) -docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*' +```bash +DB=/meta-core/db/meta-core.db # dev stack: /meta-core-db/meta-core.db +Q() { docker exec metacore-app sqlite3 -readonly "$DB" "$1"; } + +# Roots +Q "SELECT count(*) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0'" +Q "SELECT substr(k,11) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0' LIMIT 10" + +# Full document for one root +Q "SELECT k,v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'" # One property -docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration' - -# Confirm keys are STRING (not HASH) -docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath' -# string +Q "SELECT v FROM kv WHERE k = 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'" # Reverse-index lookup (bare CID — no algorithm prefix) -docker exec metacore-app redis-cli GET 'cid:bafkreih...' # → uuid +Q "SELECT v FROM kv WHERE k = 'cid:bafkreih...'" # → uuid -# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work) -docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*' +# The cids key-set +Q "SELECT k FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids0'" # canonical_cid is NOT stored. Derive it via the API instead: curl -k https://metacore-dev.localhost:8083/api/meta/ | jq '.canonical_cid, .cids' # Schema-version sentinel -docker exec metacore-app redis-cli GET 'meta-core:schema-version' +Q "SELECT v FROM kv WHERE k = 'meta-core:schema-version'" # Streams (server-side; external consumers should use SSE) -docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10 -docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10 +Q "SELECT k,v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10" # HTTP equivalents — preferred curl -k 'https://metacore-dev.localhost:8083/api/meta/' # CID → document @@ -286,12 +364,14 @@ curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats | Component | Path | |---|---| -| Redis client + flat-key ops | `internal/storage/client.go` | +| Storage engine (SQLite kv table, writer, group commit) | `internal/kv/kv.go` | +| Event streams as kv rows | `internal/kv/stream.go` | +| Flat-key record ops | `internal/storage/client.go` | +| Search (field index + FTS) | `internal/storage/search.go` | | UUIDv7 minting | `internal/storage/uuid.go` | -| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` | +| CID reverse index + aliases + canonical-CID derivation | `internal/storage/cid_resolution.go` | | Schema-version gate | `internal/storage/schema_version.go` | -| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` | -| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` | +| Boot-time `meta:events` republish | `internal/events/meta_publisher.go` | | Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` | | SSE event mirror | `internal/api/sse_events.go` | | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) | diff --git a/docs/uuid-rooted-metadata.md b/docs/uuid-rooted-metadata.md index cb33b1b..529e5d4 100644 --- a/docs/uuid-rooted-metadata.md +++ b/docs/uuid-rooted-metadata.md @@ -20,6 +20,12 @@ > `cid:` → uuid. A CIDv1's multicodec already names its hash > function, so the `:` prefix was redundant. > +> 3. **The store is no longer Redis (2026-10).** The same keys now live in +> one string→string table of an embedded SQLite file; Redis SETs became +> one key per member (`file:__index__` → `idx:roots/`, +> `file:/duplicates` → `dup:/`). Redis types named +> below are the shape at the time. +> > Text below that references either shape describes the *proposal*, not the > code. The rank ordering and everything else did ship as written. @@ -135,7 +141,7 @@ sole writer is meta-core itself: | Key | Type (as shipped) | Purpose | |-----|------|---------| | `cids/` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `:` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. | -| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. | +| `duplicates` | Redis SET at the time; since 2026-10 one `dup:/` key per path | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. | | ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. | Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`) @@ -148,7 +154,8 @@ for using sets where they're the right structure.~~ **Reversed for `cids`.** The all-strings convention won: `cids` ships as flat `cids/` STRING keys so that two peers merging the same record can never disagree about set membership. `duplicates` remained a Redis SET (it is leader-local and never -merged across peers). +merged across peers) until the 2026-10 SQLite switch, which flattened it to +`dup:/` keys. ### Canonical CID selection diff --git a/editor/src/api/snapshotApi.ts b/editor/src/api/snapshotApi.ts index c0e5af5..fcc79a9 100644 --- a/editor/src/api/snapshotApi.ts +++ b/editor/src/api/snapshotApi.ts @@ -81,7 +81,7 @@ export class SnapshotAPI { } /** - * Wipe Redis metadata. Pass scope=metadata. + * Wipe all stored metadata. Pass scope=metadata. */ static async wipe(scope: 'metadata'): Promise { const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, { diff --git a/editor/src/components/SnapshotPanel.tsx b/editor/src/components/SnapshotPanel.tsx index 0071025..e984596 100644 --- a/editor/src/components/SnapshotPanel.tsx +++ b/editor/src/components/SnapshotPanel.tsx @@ -43,7 +43,7 @@ export function SnapshotPanel() { const runWipe = async () => { const ok = window.confirm( - 'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' + + 'This deletes ALL stored metadata. Files on disk are not touched, but every manual ' + 'edit you made via this editor will be lost. Continue?' ); if (!ok) return; diff --git a/go.mod b/go.mod index 8a3c37e..4f4a7c5 100644 --- a/go.mod +++ b/go.mod @@ -1,22 +1,25 @@ module github.com/metazla/meta-core -go 1.21 +go 1.26.0 require ( - github.com/alicebob/miniredis/v2 v2.31.1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/mr-tron/base58 v1.3.0 - github.com/redis/go-redis/v9 v9.4.0 golang.org/x/crypto v0.31.0 golang.org/x/net v0.21.0 + modernc.org/sqlite v1.60.1 ) require ( - github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/yuin/gopher-lua v1.1.0 // indirect - golang.org/x/sys v0.28.0 + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) + +require golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 0eb8064..42ba8dc 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,62 @@ -github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= -github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= -github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y= -github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg= -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= -github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= -github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= -github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= -github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw= +modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/api/admin_migrate.go b/internal/api/admin_migrate.go deleted file mode 100644 index 13247f7..0000000 --- a/internal/api/admin_migrate.go +++ /dev/null @@ -1,131 +0,0 @@ -package api - -import ( - "net/http" - "strconv" -) - -// handleMigrateDualRoots triggers the one-shot sweep that re-unifies -// stranded midhash-rooted entries into their UUID roots. See -// storage.MigrateDualRoots for the data-flow detail and -// docs/api-mediated-access.md for the bug background. -// -// Returns {fixed: N} on success. Safe to call repeatedly — entries already -// migrated are skipped on the next pass. -func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - fixed, err := s.storage.MigrateDualRootsWithTimeout() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "fixed": fixed, - }) -} - -// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the -// retired `domain=film|tv` values to `screen` and backfills `workForm` from -// `contentKind` (METADATA_KEYS.md §14.17). See -// storage.MigrateDomainScreen for why the vocabulary change is paid in the -// data rather than with a read-side alias. -// -// Returns {fixed: N} — records changed, not records examined. Safe to call -// repeatedly: a record already on the current vocabulary is skipped. -func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - fixed, err := s.storage.MigrateDomainScreenWithTimeout() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "fixed": fixed, - }) -} - -// handleSweepLiteratureEchoes removes query echoes stored on literature cards: -// `anilistid` on a site card, and `workcid` / position keys on any card -// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for -// why chapters and anchor cards' ids are deliberately left alone. -// -// Dry-run unless `?apply=true`: unlike the migrations above this one deletes -// keys, so a bare call only reports what it would delete. An unparseable -// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a -// sweep that ran. -// -// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count -// records, byKey counts fields per key. Safe to call repeatedly: a swept -// record plans nothing on the next pass. -func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - apply := false - if v := r.URL.Query().Get("apply"); v != "" { - parsed, err := strconv.ParseBool(v) - if err != nil { - writeError(w, http.StatusBadRequest, "apply must be true or false") - return - } - apply = parsed - } - - res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply) - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "dryRun": !apply, - "planned": res.Planned, - "fixed": res.Fixed, - "byKey": res.ByKey, - }) -} - -// handleMigrateUserCids triggers the one-shot sweep that builds -// `udl:idx:user::cids` for User Data Layer records written before that -// index existed. See storage.UDLBackfillUserCids. -// -// The index is what makes whole-profile export a set read instead of a -// keyspace scan, and it is maintained by the upsert script from now on — so -// this only ever matters once, for data already on disk. Until it has run, a -// profile that predates the index exports as **empty**, which means a -// cross-instance recovery reports success having restored nothing. -// -// Returns {fixed: N} — cids added, not cids examined. Safe to call repeatedly: -// a second pass adds nothing. -func (s *Server) handleMigrateUserCids(w http.ResponseWriter, r *http.Request) { - if !s.storage.IsConnected() { - writeError(w, http.StatusServiceUnavailable, "storage not connected") - return - } - - fixed, err := s.storage.UDLBackfillUserCids() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - writeJSON(w, http.StatusOK, map[string]interface{}{ - "success": true, - "fixed": fixed, - }) -} diff --git a/internal/api/admin_sweep_test.go b/internal/api/admin_sweep_test.go deleted file mode 100644 index 9da3ffa..0000000 --- a/internal/api/admin_sweep_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/alicebob/miniredis/v2" - - "github.com/metazla/meta-core/internal/storage" -) - -// newSweepTestServer is a Server over miniredis with one literature card -// carrying a copied `workcid`. -func newSweepTestServer(t *testing.T) *Server { - t.Helper() - mr := miniredis.RunT(t) - c := storage.NewClient("") - if err := c.Connect("redis://" + mr.Addr()); err != nil { - t.Fatalf("connect: %v", err) - } - t.Cleanup(func() { _ = c.Close() }) - for k, v := range map[string]string{ - "domain": "literature", - "fileType": "card", - "workcid": "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy", - } { - if err := c.SetProperty("card-1", k, v); err != nil { - t.Fatalf("seed %s: %v", k, err) - } - } - return &Server{storage: c} -} - -type sweepResponse struct { - Success bool `json:"success"` - DryRun bool `json:"dryRun"` - Planned int `json:"planned"` - Fixed int `json:"fixed"` - ByKey map[string]int `json:"byKey"` -} - -func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) { - t.Helper() - rr := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil) - s.handleSweepLiteratureEchoes(rr, req) - var body sweepResponse - if rr.Code == http.StatusOK { - if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { - t.Fatalf("decode %s: %v", rr.Body.String(), err) - } - } - return rr, body -} - -// A bare call must never delete: the default is the dry run. -func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) { - s := newSweepTestServer(t) - - rr, body := postSweep(t, s, "") - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String()) - } - if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 { - t.Errorf("dry run = %+v", body) - } - if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" { - t.Errorf("dry run deleted workcid") - } - - rr, body = postSweep(t, s, "?apply=true") - if rr.Code != http.StatusOK { - t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String()) - } - if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 { - t.Errorf("apply = %+v", body) - } - if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" { - t.Errorf("apply left workcid = %q", v) - } -} - -// A typo must not pass for a sweep that ran. -func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) { - s := newSweepTestServer(t) - - rr, _ := postSweep(t, s, "?apply=yes-please") - if rr.Code != http.StatusBadRequest { - t.Errorf("status = %d, want 400", rr.Code) - } -} diff --git a/internal/api/files_register_test.go b/internal/api/files_register_test.go index 8817097..7c049a1 100644 --- a/internal/api/files_register_test.go +++ b/internal/api/files_register_test.go @@ -9,24 +9,16 @@ import ( "strings" "testing" - "github.com/alicebob/miniredis/v2" "github.com/metazla/meta-core/internal/config" - "github.com/metazla/meta-core/internal/storage" ) -// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a -// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs — +// newRegisterTestServer wires handleRegisterFile to a temp store and a temp +// FILES_PATH. Built directly, not through NewServer, so no watcher runs — // which is exactly the configuration this route exists for. -func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) { +func newRegisterTestServer(t *testing.T) (*Server, string) { t.Helper() - mr := miniredis.RunT(t) - c := storage.NewClient("") - if err := c.Connect("redis://" + mr.Addr()); err != nil { - t.Fatalf("connect to miniredis: %v", err) - } - t.Cleanup(func() { _ = c.Close() }) files := t.TempDir() - return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr + return &Server{storage: newTestStorage(t), config: &config.Config{FilesPath: files}}, files } func writeFixture(t *testing.T, files, rel, body string) string { @@ -52,20 +44,20 @@ func postRegister(t *testing.T, s *Server, path string) (int, registerFileRespon return rr.Code, resp } -func streamLen(t *testing.T, mr *miniredis.Miniredis) int { +func streamLen(t *testing.T, s *Server) int { t.Helper() - entries, err := mr.Stream("file:events") + n, err := s.storage.GetStreamLength("file:events") if err != nil { - return 0 + t.Fatalf("stream length: %v", err) } - return len(entries) + return int(n) } // The whole point of the route: with the watcher off, a file written under // plugin/ becomes a record resolvable by its midhash, and meta-share hears // about it on file:events. func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) { - s, files, mr := newRegisterTestServer(t) + s, files := newRegisterTestServer(t) rel := "plugin/subtitles/ab12cd34.eng.srt" full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n") @@ -85,20 +77,19 @@ func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) { t.Fatalf("filePath = %q, want %q", fp, full) } - entries, err := mr.Stream("file:events") + entries, err := s.storage.XRevRange("file:events", "+", "-", 0) if err != nil || len(entries) != 1 { t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err) } - vals := strings.Join(entries[0].Values, "|") - if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) { - t.Fatalf("event must carry the midhash and the relative path, got %q", vals) + if vals := entries[0].Values; vals["midhash256"] != resp.CID || vals["path"] != rel { + t.Fatalf("event must carry the midhash and the relative path, got %v", vals) } } // Re-uploading the same subtitle is a no-op: same root, no second event (which // would make meta-share re-chunk and re-provide the file). func TestRegisterFile_IsIdempotent(t *testing.T) { - s, files, mr := newRegisterTestServer(t) + s, files := newRegisterTestServer(t) rel := "plugin/subtitles/same.eng.srt" writeFixture(t, files, rel, "same bytes") @@ -110,7 +101,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) { if second.Root != first.Root || second.CID != first.CID { t.Fatalf("second register must return the same record: %+v vs %+v", second, first) } - if n := streamLen(t, mr); n != 1 { + if n := streamLen(t, s); n != 1 { t.Fatalf("an idempotent re-register must not publish again, stream has %d", n) } } @@ -118,7 +109,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) { // Only plugin output may be registered here. /files/watch is the user's // library and belongs to the watcher; traversal must not escape either. func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) { - s, files, _ := newRegisterTestServer(t) + s, files := newRegisterTestServer(t) writeFixture(t, files, "watch/movie.mkv", "x") for _, p := range []string{ "", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv", @@ -131,7 +122,7 @@ func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) { } func TestRegisterFile_MissingFileAndDirectory(t *testing.T) { - s, files, _ := newRegisterTestServer(t) + s, files := newRegisterTestServer(t) if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound { t.Errorf("missing file: want 404, got %d", code) } diff --git a/internal/api/files_tuples_test.go b/internal/api/files_tuples_test.go index d38c91d..caa6ee8 100644 --- a/internal/api/files_tuples_test.go +++ b/internal/api/files_tuples_test.go @@ -7,18 +7,12 @@ import ( "sort" "testing" - "github.com/alicebob/miniredis/v2" "github.com/metazla/meta-core/internal/storage" ) func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) { t.Helper() - mr := miniredis.RunT(t) - c := storage.NewClient("") - if err := c.Connect("redis://" + mr.Addr()); err != nil { - t.Fatalf("connect to miniredis: %v", err) - } - t.Cleanup(func() { _ = c.Close() }) + c := newTestStorage(t) return &Server{storage: c}, c } @@ -105,7 +99,7 @@ func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) { } func TestFileTuples_StorageNotConnected(t *testing.T) { - s := &Server{storage: storage.NewClient("")} + s := &Server{storage: storage.NewClient()} code, _ := getFileTuples(t, s, "") if code != http.StatusServiceUnavailable { diff --git a/internal/api/identity_gate_test.go b/internal/api/identity_gate_test.go index 94dd7de..7bc1d56 100644 --- a/internal/api/identity_gate_test.go +++ b/internal/api/identity_gate_test.go @@ -1,6 +1,6 @@ package api -// End-to-end exercise of the account gate against a live router + miniredis: +// End-to-end exercise of the account gate against a live router + a temp store: // generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify // both the key and every row are gone. Deliberately drives the real HTTP // handlers rather than the packages beneath them, because the bugs that matter @@ -16,21 +16,14 @@ import ( "net/http/httptest" "testing" - "github.com/alicebob/miniredis/v2" "github.com/gorilla/mux" "github.com/metazla/meta-core/internal/config" "github.com/metazla/meta-core/internal/identity" - "github.com/metazla/meta-core/internal/storage" ) func newTestServer(t *testing.T) *Server { t.Helper() - mr := miniredis.RunT(t) - stor := storage.NewClient("") - if err := stor.Connect("redis://" + mr.Addr()); err != nil { - t.Fatalf("connect: %v", err) - } - t.Cleanup(func() { _ = stor.Close() }) + stor := newTestStorage(t) s := &Server{ config: &config.Config{MetaCorePath: t.TempDir()}, diff --git a/internal/api/kv.go b/internal/api/kv.go index fc3d87b..7c4b3df 100644 --- a/internal/api/kv.go +++ b/internal/api/kv.go @@ -206,10 +206,10 @@ type KVSearchResponse struct { // KVFindResponse is the response for GET /api/kv/find type KVFindResponse struct { - Contains string `json:"contains"` - Fields []string `json:"fields"` + Contains string `json:"contains"` + Fields []string `json:"fields"` Matches []storage.FindMatch `json:"matches"` - Truncated bool `json:"truncated"` + Truncated bool `json:"truncated"` } // KVValueResponse is the response for GET /api/kv/value diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 2db7e2d..0884501 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -40,9 +40,9 @@ type MetadataSearchRequest struct { Limit int `json:"limit,omitempty"` // Filters is the structured form: AND across entries, OR within an - // entry's Values. Evaluated in memory against the search index alongside - // the free-text Query, so an id-anchored lookup costs a map probe per - // record rather than a Redis round-trip per record. + // entry's Values. Narrowed in SQL through the field index alongside the + // free-text Query, then verified per record (see serveIndexedSearch), so an + // id-anchored lookup costs an index probe rather than a read per record. // // Property/PropertyValue is the single-filter legacy shape and folds into // this list (see normalizedFilters) with Exact=false, preserving its @@ -445,9 +445,8 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) { } // Everything else — free text, structured filters, both, or neither - // (which returns the first `limit`) — is served from the in-memory search - // index (search_index.go). Matching and the returned metadata both come - // from memory, so a query costs zero Redis round-trips. + // (which returns the first `limit`) — is served from the store's derived + // search indexes (search_index.go → storage.Search). // // There is deliberately no second path. The property branch used to fall // through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which diff --git a/internal/api/schema_handlers.go b/internal/api/schema_handlers.go index 6be5f9d..9591241 100644 --- a/internal/api/schema_handlers.go +++ b/internal/api/schema_handlers.go @@ -3,10 +3,10 @@ package api import ( "encoding/json" "net/http" - "time" ) -// handleSchemaGet returns the live per-field schema derived from meta:events. +// handleSchemaGet returns the per-field schema (recomputed from the store every +// few minutes; see schema.Indexer). func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) { if s.schemaIndexer == nil { http.Error(w, "schema indexer not running", http.StatusServiceUnavailable) @@ -17,29 +17,26 @@ func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) { _ = json.NewEncoder(w).Encode(snap) } -// handleSchemaRescan re-emits a set event for every existing property via the -// meta publisher, waits for the indexer to drain, and returns the refreshed -// schema. Equivalent to a full SCAN-based rebuild without locking Redis. +// handleSchemaRescan recomputes the schema from a full scan of the store and +// returns it. It no longer republishes every property into meta:events — the +// indexer reads the store itself — so a rescan floods no consumer. func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) { if s.schemaIndexer == nil { http.Error(w, "schema indexer not running", http.StatusServiceUnavailable) return } - count, err := s.RepublishMetadata() + snap, err := s.schemaIndexer.Rebuild(r.Context()) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - - s.schemaIndexer.WaitForDrain(10 * time.Second) - - snap := s.schemaIndexer.Snapshot() - snap.Source = "rescan" + out := *snap + out.Source = "rescan" w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ - "published_events": count, - "schema": snap, + "published_events": 0, + "schema": &out, }) } diff --git a/internal/api/search_index.go b/internal/api/search_index.go index e29f63c..7dedde5 100644 --- a/internal/api/search_index.go +++ b/internal/api/search_index.go @@ -1,221 +1,59 @@ package api import ( - "log" "net/http" - "sort" "strings" - "sync" - "sync/atomic" - "time" "github.com/metazla/meta-core/internal/storage" ) -// searchIndexRefreshInterval is how often the in-memory search index is rebuilt -// from Redis in the background. Records ingest continuously, so a newly stored -// record becomes searchable within one interval; the rebuild is a single -// batched bulk read (storage.GetSearchIndexTuples), off the request path. -const searchIndexRefreshInterval = 60 * time.Second - -// searchIndexReconcileEvery is how many refresh ticks pass between full -// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full -// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all. -const searchIndexReconcileEvery = 30 - -// searchIndex is an in-memory snapshot of every record's search haystack -// (hashID + lowercased searchable fields), kept current from Redis. It exists -// so POST /api/metadata/search matches in memory (string ops over a slice) -// instead of doing a SCAN+MGET Redis round-trip per record — the latter took -// 40s+ over ~72k records and made a live parallel search path unusable. +// serveIndexedSearch answers a search from the database's derived indexes +// (storage.Search): SQL narrows to a superset of candidates through the field +// expression index and the FTS5 trigram table, then every candidate is checked +// here with the exact matchers below — a token-AND match against the record's +// haystack (or a whole-query hashID substring, preserving the legacy +// behaviour) AND every structured filter. // -// # Two update paths, and why -// -// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a -// complete fresh copy. That is O(corpus) regardless of how much changed, and it -// used to run every 60 s forever. Measured on a 19 622-record staging corpus -// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side -// held 783 MB for data Redis stores in 135 MB — largely the double copy. -// -// `refreshIncremental` re-reads only the records that changed, tracked by -// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady -// state; the full rebuild runs once at startup (there is nothing to be -// incremental against) and then only as an explicit repair. -// -// The distinction matters because meta-core is the platform's permanent, -// additive store — every search hit on every gateway is retained — so a rebuild -// priced against the whole corpus is a cost that only ever grows. -type searchIndex struct { - mu sync.RWMutex - // byID is the authoritative content; tuples is a stable-order view of it - // rebuilt on each update. Map iteration order in Go is randomised, so - // serving a limited query straight off the map would return a different - // subset each call — the slice is what keeps results stable. - byID map[string]storage.SearchTuple - tuples []storage.SearchTuple - ready atomic.Bool -} - -// snapshot returns the current tuples slice and whether the index has built at -// least once. Every update swaps in a fresh slice under the write lock (never -// mutates in place), so the returned slice is safe to read lock-free. -func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) { - idx.mu.RLock() - defer idx.mu.RUnlock() - return idx.tuples, idx.ready.Load() -} - -// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds -// the write lock. -func (idx *searchIndex) rebuildViewLocked() { - view := make([]storage.SearchTuple, 0, len(idx.byID)) - for _, t := range idx.byID { - view = append(view, t) - } - // Stable across rebuilds so a limited query returns a consistent page - // rather than a fresh random subset every interval. - sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID }) - idx.tuples = view -} - -// refresh rebuilds the whole index from Redis (a batched bulk read). Used for -// the initial build and as an explicit repair — not the steady state. -func (idx *searchIndex) refresh(stor *storage.Client) { - if stor == nil || !stor.IsConnected() { - return - } - tuples, err := stor.GetSearchIndexTuples() - if err != nil { - log.Printf("[API] search index refresh failed: %v", err) - return - } - byID := make(map[string]storage.SearchTuple, len(tuples)) - for _, t := range tuples { - byID[t.HashID] = t - } - idx.mu.Lock() - idx.byID = byID - idx.rebuildViewLocked() - idx.mu.Unlock() - idx.ready.Store(true) -} - -// refreshIncremental drains the dirty set and patches only those records. -// Returns whether more dirty entries remain, so a burst is caught up over -// consecutive passes rather than one per interval. -// -// A record in the dirty set but absent from storage was deleted and is evicted -// here — otherwise it would keep answering searches until the next full -// rebuild. A batch that fails after the (destructive) drain is re-marked by -// storage so it is retried rather than silently lost. -func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) { - if stor == nil || !stor.IsConnected() { - return false - } - ids, more, err := stor.DrainDirty() - if err != nil { - log.Printf("[API] search index drain failed: %v", err) - return false - } - if len(ids) == 0 { - return false - } - tuples, gone, err := stor.GetSearchIndexTuplesFor(ids) - if err != nil { - stor.MarkDirty(ids...) - log.Printf("[API] incremental search index refresh failed: %v", err) - return false - } - - idx.mu.Lock() - if idx.byID == nil { - idx.byID = make(map[string]storage.SearchTuple, len(tuples)) - } - for _, t := range tuples { - idx.byID[t.HashID] = t - } - for _, id := range gone { - delete(idx.byID, id) - } - idx.rebuildViewLocked() - idx.mu.Unlock() - - log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples)) - return more -} - -// runRefreshLoop builds the index once in full, then keeps it current from the -// dirty set, with a periodic full reconcile as a safety net. -// -// The reconcile exists because a *missed* dirty mark is silent: no error, just -// a record whose indexed copy quietly diverges from storage until something -// forces a rebuild. Marking is spread across eight write paths plus anything -// added later, and the cost of getting it wrong is invisible — so the index -// should not depend on that coverage being perfect forever. Every -// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs -// whatever drifted. -// -// That still drops the full rebuild from every 60 s to every 30 min — a ~30× -// reduction in the steady-state cost that measured at ~45 % of all Redis CPU — -// while keeping the self-healing property that made the naive loop safe. -func (idx *searchIndex) runRefreshLoop(stor *storage.Client) { - idx.refresh(stor) - ticker := time.NewTicker(searchIndexRefreshInterval) - defer ticker.Stop() - ticks := 0 - for range ticker.C { - ticks++ - if ticks%searchIndexReconcileEvery == 0 { - idx.refresh(stor) - continue - } - // Drain until caught up, so a burst larger than one batch does not take - // one interval per batch to land. - for idx.refreshIncremental(stor) { - } - } -} - -// serveIndexedSearch answers a search from the in-memory index: a token-AND -// match against each record's haystack (or a whole-query hashID substring, -// preserving the legacy behaviour) AND every structured filter, then returns -// the full metadata for the <=limit survivors. No Redis round-trip per scanned -// record — matching and results both come from memory. +// Until 2026-10 this ran against an in-memory copy of every record, rebuilt +// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh) +// and is gone. The matchers did not change, so neither did the semantics. // // This is the ONLY search path. Free text, filters, both, or neither all land // here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it. func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) { - tuples, ready := s.search.snapshot() - if !ready { - // First call before the background build finished: warm inline with the - // efficient batched read (seconds), never the per-record SCAN fallback. - s.search.refresh(s.storage) - tuples, _ = s.search.snapshot() - } tokens := strings.Fields(strings.ToLower(req.Query)) queryLower := strings.ToLower(req.Query) filters := normalizedFilters(req) - results := make([]MetadataSearchResult, 0, req.Limit) - for _, t := range tuples { - if len(results) >= req.Limit { - break - } + + narrow := make([]storage.SearchFilter, 0, len(filters)) + for _, f := range filters { + narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values}) + } + hits, err := s.storage.Search(req.Query, narrow, req.Limit, func(hashID string, fields map[string]string) bool { if req.Query != "" && - !matchAllTokens(t.Haystack, tokens) && - !strings.Contains(strings.ToLower(t.HashID), queryLower) { - continue + !matchAllTokens(storage.Haystack(fields), tokens) && + !strings.Contains(strings.ToLower(hashID), queryLower) { + return false } - if !recordMatchesFilters(t.Fields, filters) { - continue - } - // Full metadata is cached in the index — return it straight from memory, - // no Redis round-trip per result. - results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields}) + return recordMatchesFilters(fields, filters) + }) + if err != nil { + writeError(w, http.StatusInternalServerError, "search failed: "+err.Error()) + return + } + total, err := s.storage.CountRoots() + if err != nil { + total = 0 + } + + results := make([]MetadataSearchResult, 0, len(hits)) + for _, h := range hits { + results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields}) } writeJSON(w, http.StatusOK, MetadataSearchResponse{ Results: results, Count: len(results), - Total: len(tuples), + Total: int(total), }) } diff --git a/internal/api/server.go b/internal/api/server.go index e3661b9..7717994 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -39,7 +39,6 @@ type Server struct { metaPublisher *events.MetaPublisher schemaIndexer *schema.Indexer webdavHandler *webdav.Handler - search *searchIndex // challenges backs proof-of-possession on reveal/delete. In-process, so a // restart invalidates the ones in flight — see identity/challenge.go. challenges *identity.ChallengeStore @@ -51,7 +50,7 @@ type Server struct { router *mux.Router server *http.Server // stats caches the keyspace-walk half of GET /api/stats so a polling - // dashboard never puts a multi-second Redis walk on the request path. + // dashboard never puts a multi-second store walk on the request path. stats *statsCache } @@ -71,7 +70,6 @@ func NewServer( config: cfg, leaderProvider: leaderProvider, storage: stor, - search: &searchIndex{}, router: mux.NewRouter(), stats: &statsCache{}, } @@ -80,7 +78,7 @@ func NewServer( var scanFunc mounts.ScanFunc if cfg.EnableFileWatcher { s.watcherDispatcher = watcher.NewDispatcher() - // Wire storage client for Redis Stream publishing + // Wire storage client for file:events publishing if stor != nil { s.watcherDispatcher.SetStorageClient(stor) } @@ -90,7 +88,7 @@ func NewServer( } else { s.fileWatcher = fileWatcher // Wire storage so the watcher can persist (path,size,mtime,cid) - // tuples to Redis as it computes midhashes. + // tuples to the store as it computes midhashes. if stor != nil { fileWatcher.SetStorageClient(stor) } @@ -112,7 +110,7 @@ func NewServer( // Pass republish callback to handlers (handles nil metaPublisher case) s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata) - // Hydrate the in-memory state registry from Redis tuples so + // Hydrate the in-memory state registry from stored tuples so // the first scan tick after boot can skip ComputeMidHash256 // for files whose (size, mtime) match. configs := watchersManager.List() @@ -123,7 +121,7 @@ func NewServer( if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil { log.Printf("[API] Warning: failed to hydrate watcher state: %v", err) } else { - log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped) + log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped) } } } @@ -272,20 +270,8 @@ func (s *Server) setupRoutes() { // without going through Authelia. s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET") - // Admin: reunify stranded midhash-rooted entries into their UUID roots - // (the dual-root pattern caused by historical writes that bypassed CID - // resolution). - s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST") - // One-shot vocabulary sweep: domain film|tv → screen, workForm backfill - // (METADATA_KEYS.md §14.17). Idempotent. - s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST") - s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST") - // One-shot sweep of query echoes on literature cards (meta-read - // reading-model.md §7). Dry-run unless ?apply=true. Idempotent. - s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST") - - // SSE event streams. Mediated mirror of the Redis Streams; consumers - // reach them over HTTP instead of touching Redis directly. Auth-bypass + // SSE event streams. The only way consumers see meta-core's change + // streams — they never touch the database. Auth-bypass // applies but exposure should be inside-only (do NOT proxy through // Caddy publicly). See docs/api-mediated-access.md "Auth". s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET") @@ -363,44 +349,31 @@ func (s *Server) Start() error { s.mountStatsPoller.Start() } - // Build + periodically refresh the in-memory search index so - // POST /api/metadata/search matches in memory instead of a Redis - // round-trip per record (see search_index.go). - if s.storage != nil { - go s.search.runRefreshLoop(s.storage) - } - - // Start meta publisher (if storage is connected) - // Publishes keyspace notifications to meta:events stream + // Start meta publisher (if storage is connected). Live change events are + // written by the storage engine itself; the publisher owns the bulk + // republish below. if s.storage != nil && s.storage.IsConnected() { - s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient()) + s.metaPublisher = events.NewMetaPublisher(s.storage) if err := s.metaPublisher.Start(); err != nil { log.Printf("[API] Warning: failed to start meta publisher: %v", err) } else { - // Schema indexer consumes the same stream to derive a live - // per-field schema (type/hint/breakdown). Start it before the - // republish flood so it captures the bootstrap events. - s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix()) + // Schema indexer: per-field schema (type/hint/breakdown) from a + // periodic scan of the store. + s.schemaIndexer = schema.NewIndexer(s.storage.Store()) if err := s.schemaIndexer.Start(); err != nil { log.Printf("[API] Warning: failed to start schema indexer: %v", err) s.schemaIndexer = nil } - // Republish events for all existing metadata + // Republish events for all existing metadata. Consumers that + // rebuild by replaying meta:events from the beginning (meta-fuse) + // depend on it. go func() { - // Build any missing per-record field indexes first, in ONE - // keyspace pass. The republish below reads every record, and - // an un-indexed record read costs a full-keyspace SCAN — on a - // real store that walk warmed at ~1.3 records/s (a ~2.6h tail, - // paid again on every restart). Warming first turns the whole - // republish into index-served reads. Idempotent, so a - // fully-indexed store just pays the single scan. - // See storage/field_index.go. started := time.Now() - if n, err := s.storage.WarmFieldIndexes(); err != nil { - log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err) + if n, err := s.storage.RebuildSearchIfEmpty(); err != nil { + log.Printf("[API] Warning: search row rebuild failed: %v", err) } else if n > 0 { - log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond)) + log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond)) } count, err := s.RepublishMetadata() diff --git a/internal/api/snapshot.go b/internal/api/snapshot.go index 9bd962a..a491b66 100644 --- a/internal/api/snapshot.go +++ b/internal/api/snapshot.go @@ -103,7 +103,7 @@ func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, result) } -// handleSnapshotWipe clears Redis metadata. +// handleSnapshotWipe clears all stored metadata. // POST /api/snapshot/wipe?scope=metadata func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) { if !s.storage.IsConnected() { diff --git a/internal/api/sse_events.go b/internal/api/sse_events.go index 8a72dcd..b063d0f 100644 --- a/internal/api/sse_events.go +++ b/internal/api/sse_events.go @@ -3,7 +3,6 @@ package api import ( "context" "encoding/json" - "errors" "fmt" "log" "net/http" @@ -11,15 +10,17 @@ import ( "strings" "time" - "github.com/redis/go-redis/v9" + "github.com/metazla/meta-core/internal/kv" ) -// SSE event stream endpoints for the file:events and meta:events Redis Streams. +// SSE event stream endpoints for the file:events and meta:events streams. // // Wire contract (see docs/api-mediated-access.md): -// - One SSE event per Redis Stream entry. -// - `id:` is the opaque Redis Stream entry ID (`-`); consumers -// MUST treat it as a black-box string they echo back via Last-Event-ID. +// - One SSE event per stream entry. +// - `id:` is the opaque stream entry ID (`-`, the Redis Streams +// form, kept across the move to SQLite so cursors consumers persisted to +// disk still compare correctly); consumers MUST treat it as a black-box +// string they echo back via Last-Event-ID. // - `event:` is the `type` field from the underlying entry. // - `data:` is the JSON-encoded rest of the entry (all fields except `type`). // - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence. @@ -30,30 +31,30 @@ import ( const ( sseHeartbeatInterval = 30 * time.Second - // sseReadBlock controls how long each XREAD blocks before returning. + // sseReadBlock controls how long each read waits for new entries. // Much shorter than sseHeartbeatInterval so the handler stays responsive // to client disconnects and the heartbeat check runs frequently enough // that the keep-alive fires within a few seconds of its nominal cadence. sseReadBlock = 2 * time.Second - // sseReadCount caps how many entries one XREAD returns. Small enough + // sseReadCount caps how many entries one read returns. Small enough // to keep per-flush memory low, large enough to drain bursts cheaply. sseReadCount = 100 ) -// handleEventsFiles streams the file:events Redis Stream as SSE. +// handleEventsFiles streams the file:events stream as SSE. func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) { s.streamSSE(w, r, "file:events") } -// handleEventsMeta streams the meta:events Redis Stream as SSE. +// handleEventsMeta streams the meta:events stream as SSE. func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) { s.streamSSE(w, r, "meta:events") } -// streamSSE is the shared SSE pump shared by both endpoints. One handler per -// stream — there is no fan-out; each subscriber holds its own XREAD loop. -// That's fine at our scale (≤20 subscribers; one per consumer service). +// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads +// the stream itself and sleeps on the store's commit broadcast between reads — +// fine at our scale (≤20 subscribers; one per consumer service). func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) { if !s.storage.IsConnected() { w.Header().Set("Retry-After", "5") @@ -89,14 +90,14 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string w.WriteHeader(http.StatusOK) flusher.Flush() - client := s.storage.GetRedisClient() - if client == nil { + store := s.storage.Store() + if store == nil { // Storage was just disconnected — close cleanly. return } ctx := r.Context() - cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher) + cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher) lastWrite := time.Now() for { @@ -104,37 +105,34 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string return } - // Bounded BLOCK lets us re-check ctx and emit heartbeats. The - // context's deadline isn't propagated into XREAD's BLOCK arg — - // that's a poll-style timeout on Redis itself — so we keep BLOCK - // short. - readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second) - result, err := client.XRead(readCtx, &redis.XReadArgs{ - Streams: []string{stream, cursor}, - Block: sseReadBlock, - Count: sseReadCount, - }).Result() - cancel() - + // Take the wake-up channel BEFORE reading, so an append landing + // between the read and the wait still wakes us. + wake := store.Wait(stream) + entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount) if err != nil { - // redis.Nil = BLOCK timeout with no entries. Anything else - // while the client is still connected is a real failure. - if errors.Is(err, redis.Nil) || isContextTimeout(err) { - // fall through to heartbeat check - } else if errors.Is(err, context.Canceled) || ctx.Err() != nil { - return - } else { - log.Printf("[SSE %s] XREAD error: %v", stream, err) + if ctx.Err() != nil { return } + log.Printf("[SSE %s] read error: %v", stream, err) + return } - for _, str := range result { - for _, msg := range str.Messages { - writeSSEEntry(w, flusher, msg) - cursor = msg.ID - lastWrite = time.Now() + for _, e := range entries { + writeSSEEntry(w, flusher, e) + cursor = e.ID + lastWrite = time.Now() + } + + if len(entries) == 0 { + timer := time.NewTimer(sseReadBlock) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-wake: + case <-timer.C: } + timer.Stop() } // Heartbeat if quiet for >= the interval. Comment line; SSE @@ -150,38 +148,44 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string } // resolveStartCursor decides where to start reading. -// - No Last-Event-ID → start at "$" (only new entries). +// - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only +// entries appended from now on). Redis' "$" was re-resolved on every +// XREAD, which dropped anything appended between two reads. // - Last-Event-ID provided AND still in stream → resume from it. // - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event // and resume from the oldest available entry's predecessor "0", so the -// next XREAD picks up everything still in the stream. +// next read picks up everything still in the stream. // -// The "still in stream" check is cheap: one XRANGE for the first entry. -func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string { - if lastID == "" { - return "$" - } - - // One-entry XRANGE to find the oldest ID currently in the stream. +// The "still in stream" check is cheap: one read of the first entry. +func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string { xCtx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() - entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result() + + if lastID == "" { + head, err := store.LastID(xCtx, stream) + if err != nil { + log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err) + return "0" + } + return head + } + + oldestEntry, ok, err := store.Oldest(xCtx, stream) if err != nil { // Treat any error as "stream not available right now" and resume - // from the cursor — XREAD will either find it or block until new - // entries arrive. - log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err) + // from the cursor — the read either finds it or waits for new entries. + log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err) return lastID } - if len(entries) == 0 { + if !ok { // Stream is empty. Cursor may be from a previous incarnation; safest - // is to resume from the cursor — XREAD against an empty stream - // returns nothing and blocks until new entries arrive. + // is to resume from the cursor — reading an empty stream returns + // nothing and waits until new entries arrive. return lastID } - oldest := entries[0].ID + oldest := oldestEntry.ID if compareStreamIDs(lastID, oldest) >= 0 { // Cursor is at or past the oldest — fine to resume. return lastID @@ -197,39 +201,35 @@ func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastI fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload)) flusher.Flush() - // Return "0" so the next XREAD picks up entries starting from the + // Return "0" so the next read picks up entries starting from the // stream's true minimum — including `oldest` itself. return "0" } -// writeSSEEntry serialises one Redis Stream entry as an SSE event. +// writeSSEEntry serialises one stream entry as an SSE event. // // `event:` comes from the entry's `type` field (no prefix munging). `data:` -// is a JSON object containing all OTHER fields. Field values from the Redis -// stream are interface{} (typically strings); we encode whatever the stream -// holds. -func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) { +// is a JSON object containing all OTHER fields. +func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) { eventType := "message" - data := make(map[string]interface{}, len(msg.Values)) - for k, v := range msg.Values { + data := make(map[string]string, len(e.Values)) + for k, v := range e.Values { if k == "type" { - if s, ok := v.(string); ok { - eventType = s - continue - } + eventType = v + continue } data[k] = v } payload, err := json.Marshal(data) if err != nil { - log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err) + log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err) return } // Spec: each event ends with a blank line. id/event/data lines are // terminated with single \n. - fmt.Fprintf(w, "id: %s\n", msg.ID) + fmt.Fprintf(w, "id: %s\n", e.ID) fmt.Fprintf(w, "event: %s\n", eventType) fmt.Fprintf(w, "data: %s\n\n", string(payload)) flusher.Flush() @@ -263,18 +263,3 @@ func parseStreamID(id string) (int64, int64) { } return ms, seq } - -// isContextTimeout returns true for the context deadline error wrapped by -// go-redis when XREAD's read context fires. We treat it the same as -// redis.Nil — i.e. an empty poll cycle, not a fatal error. -func isContextTimeout(err error) bool { - if err == nil { - return false - } - if errors.Is(err, context.DeadlineExceeded) { - return true - } - // go-redis sometimes wraps this in its own error type. String match is - // a safety net. - return strings.Contains(err.Error(), "context deadline exceeded") -} diff --git a/internal/api/stats.go b/internal/api/stats.go index 03fcc5b..aab6fc1 100644 --- a/internal/api/stats.go +++ b/internal/api/stats.go @@ -27,8 +27,10 @@ type fileStatsJSON struct { // dashboard shows, in one request. // // Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are -// O(1) commands (SCARD, DBSIZE, INFO) plus a directory listing, recomputed on -// every call. Files and UDLUsers need a keyspace walk — ~0.5s for the file +// cheap (an index range count, a cached row count, a file stat) plus a +// directory listing, recomputed on every call. The `redis*` JSON names +// predate the move to SQLite and are kept for dashboard compatibility: +// redisKeys is the store's row count, redisMemory its on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file // tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served // from a background-refreshed cache and are null until the first sweep lands. type StatsResponse struct { @@ -47,7 +49,7 @@ type StatsResponse struct { // statsCache holds the expensive half of /api/stats. // -// Why a cache and not just "compute it": the dashboard polls. A 3s Redis walk +// Why a cache and not just "compute it": the dashboard polls. A multi-second store walk // per tick, per open tab, is a self-inflicted outage on a box that is also // serving playback — the same shape as meta-sort's /api/stats full sweep. So // the handler NEVER waits on a walk: it returns whatever the last sweep @@ -86,7 +88,7 @@ func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool { c.running = false // A failed sweep leaves the previous numbers in place rather than // blanking the tiles; sweptAt still advances so a hard-failing walk - // (Redis down) does not spin once per poll. + // (store down) does not spin once per poll. c.sweptAt = time.Now() if files != nil { c.files = files diff --git a/internal/api/storage_helper_test.go b/internal/api/storage_helper_test.go new file mode 100644 index 0000000..f318548 --- /dev/null +++ b/internal/api/storage_helper_test.go @@ -0,0 +1,19 @@ +package api + +import ( + "path/filepath" + "testing" + + "github.com/metazla/meta-core/internal/storage" +) + +// newTestStorage opens a fresh metadata store in the test's temp dir. +func newTestStorage(t *testing.T) *storage.Client { + t.Helper() + c := storage.NewClient() + if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil { + t.Fatalf("open store: %v", err) + } + t.Cleanup(func() { _ = c.Close() }) + return c +} diff --git a/internal/api/udl.go b/internal/api/udl.go index 77c767d..d5a32d6 100644 --- a/internal/api/udl.go +++ b/internal/api/udl.go @@ -7,7 +7,7 @@ package api // meta-watch's local op-log. Records arrive already signed (and, for // private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core // version-gates and indexes them but never inspects the crypto. See -// internal/storage/udl.go for the Redis model. +// internal/storage/udl.go for the key model. import ( "encoding/json" @@ -65,7 +65,7 @@ type UDLPutBatchResponse struct { Results []UDLPutBatchResult `json:"results"` } -// maxUDLBatch bounds one batch. Each record is a Redis round trip while the +// maxUDLBatch bounds one batch. Each record is a store write while the // request is held open; sized to match the sign batch on the other side of the // same operation (see maxSignBatch). const maxUDLBatch = 1000 @@ -126,7 +126,7 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) { Exists: exists, } // json.Valid guards the encoder: a malformed `value` hash field (hand-edited - // Redis, a future writer bug) would otherwise fail mid-encode and emit a + // the store, a future writer bug) would otherwise fail mid-encode and emit a // truncated body instead of a readable error. if exists && cell.HasValue && json.Valid([]byte(cell.Value)) { raw := json.RawMessage(cell.Value) diff --git a/internal/api/udl_test.go b/internal/api/udl_test.go index 86b558d..d9c3898 100644 --- a/internal/api/udl_test.go +++ b/internal/api/udl_test.go @@ -6,9 +6,6 @@ import ( "net/http/httptest" "strings" "testing" - - "github.com/alicebob/miniredis/v2" - "github.com/metazla/meta-core/internal/storage" ) // newUDLTestServer returns a Server wired to an in-memory Redis. @@ -18,14 +15,7 @@ import ( // which the UDL handlers touch — they use s.storage and nothing else. func newUDLTestServer(t *testing.T) *Server { t.Helper() - - mr := miniredis.RunT(t) - c := storage.NewClient("") - if err := c.Connect("redis://" + mr.Addr()); err != nil { - t.Fatalf("connect to miniredis: %v", err) - } - t.Cleanup(func() { _ = c.Close() }) - return &Server{storage: c} + return &Server{storage: newTestStorage(t)} } // getRecord drives handleUDLRecordGet and decodes the body into raw JSON diff --git a/internal/config/config.go b/internal/config/config.go index de72e56..ea37703 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,8 +19,8 @@ type Config struct { BaseURL string // Base URL for stable service discovery PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front) - // Redis configuration - RedisPort int // Redis port (default: 6379) + // Storage configuration + DBPathOverride string // META_CORE_DB_PATH; empty = /db/meta-core.db // HTTP API configuration HTTPPort int // HTTP API port (default: 9000) @@ -60,7 +60,7 @@ func Load() *Config { APIPort: getEnvInt("API_PORT", 8180), BaseURL: getEnv("BASE_URL", ""), PublicURL: getEnv("META_CORE_PUBLIC_URL", ""), - RedisPort: getEnvInt("REDIS_PORT", 6379), + DBPathOverride: getEnv("META_CORE_DB_PATH", ""), HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000), HTTPHost: getEnv("META_CORE_HTTP_HOST", "0.0.0.0"), HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000), @@ -83,16 +83,20 @@ func Load() *Config { return cfg } -// LockFilePath returns the path to the Redis-volume mutex. Held by +// LockFilePath returns the path to the database-volume mutex (one meta-core +// per database file). Held by // docker/leader-election.sh; nothing reads it, and it is NOT service // discovery — see docs/project-architecture/beacon-v2.md. func (c *Config) LockFilePath() string { return c.MetaCorePath + "/locks/kv-leader.lock" } -// RedisDataDir returns the path to Redis data directory -func (c *Config) RedisDataDir() string { - return c.MetaCorePath + "/db/redis" +// DBPath returns the path of meta-core's SQLite database file. +func (c *Config) DBPath() string { + if c.DBPathOverride != "" { + return c.DBPathOverride + } + return c.MetaCorePath + "/db/meta-core.db" } // MountsFilePath returns the path to the mounts configuration file diff --git a/internal/events/meta_publisher.go b/internal/events/meta_publisher.go index 69d7d5f..29cfd43 100644 --- a/internal/events/meta_publisher.go +++ b/internal/events/meta_publisher.go @@ -1,199 +1,89 @@ package events import ( - "context" "fmt" "log" - "strings" "sync" "time" - "github.com/redis/go-redis/v9" - "github.com/metazla/meta-core/internal/storage" ) const ( - // MetaEventsStream is the Redis stream name for metadata events - MetaEventsStream = "meta:events" + // MetaEventsStream is the stream name for metadata events. + MetaEventsStream = storage.MetaEventsStream // MetaEventsMaxLen is the approximate cap on stream length. Bounded // retention is a contract requirement for the SSE wire — see // docs/api-mediated-access.md "Backing-store retention and lifecycle". - // Cheap (~) trim keeps growth bounded without serialising writes. - MetaEventsMaxLen = 100_000 + MetaEventsMaxLen = storage.MetaEventsMaxLen ) -// MetaPublisher subscribes to Redis keyspace notifications -// and publishes metadata change events to the meta:events stream +// republishBatch is how many events one republish transaction appends — +// large enough to amortise the commit, small enough that the writer is never +// held for long. +const republishBatch = 5_000 + +// MetaPublisher owns the meta:events stream's bulk republish. +// +// Live change events no longer pass through here: under Redis this subscribed +// to keyspace notifications and copied each into the stream, a hop that could +// drop events whenever the subscriber lagged. The storage engine now appends +// the event in the same transaction as the change (internal/kv), so a change +// and its event commit together. type MetaPublisher struct { - client *redis.Client - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup + client *storage.Client running bool mu sync.RWMutex } -// NewMetaPublisher creates a new MetaPublisher -func NewMetaPublisher(client *redis.Client) *MetaPublisher { - ctx, cancel := context.WithCancel(context.Background()) - return &MetaPublisher{ - client: client, - ctx: ctx, - cancel: cancel, - } +// NewMetaPublisher creates a new MetaPublisher. +func NewMetaPublisher(client *storage.Client) *MetaPublisher { + return &MetaPublisher{client: client} } -// Start begins subscribing to keyspace notifications. -// -// The stream is NOT truncated on startup — the SSE wire contract requires -// Last-Event-ID resumption to survive a meta-core restart, which means -// entries must persist across restart. Growth is bounded instead by -// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent). +// Start marks the publisher ready. The stream is NOT truncated on startup — +// the SSE wire contract requires Last-Event-ID resumption to survive a +// meta-core restart. Growth is bounded by MetaEventsMaxLen instead. func (p *MetaPublisher) Start() error { p.mu.Lock() - if p.running { - p.mu.Unlock() - return nil - } + defer p.mu.Unlock() p.running = true - p.mu.Unlock() - - p.wg.Add(1) - go p.subscribeLoop() - - log.Println("[MetaPublisher] Started - listening for keyspace notifications") + log.Println("[MetaPublisher] Started") return nil } -// Stop stops the publisher +// Stop stops the publisher. func (p *MetaPublisher) Stop() error { p.mu.Lock() - if !p.running { - p.mu.Unlock() - return nil - } + defer p.mu.Unlock() p.running = false - p.mu.Unlock() - - p.cancel() - p.wg.Wait() - log.Println("[MetaPublisher] Stopped") return nil } -// IsRunning returns true if the publisher is running +// IsRunning returns true if the publisher is running. func (p *MetaPublisher) IsRunning() bool { p.mu.RLock() defer p.mu.RUnlock() return p.running } -// subscribeLoop subscribes to keyspace notifications and publishes events -func (p *MetaPublisher) subscribeLoop() { - defer p.wg.Done() - - // Subscribe to all file:* keyspace events - // Pattern: __keyspace@0__:file:* - pattern := "__keyspace@0__:file:*" - pubsub := p.client.PSubscribe(p.ctx, pattern) - defer pubsub.Close() - - log.Printf("[MetaPublisher] Subscribed to pattern: %s", pattern) - - ch := pubsub.Channel() - for { - select { - case <-p.ctx.Done(): - return - case msg := <-ch: - if msg == nil { - continue - } - // msg.Channel: __keyspace@0__:file:midhash256:abc123/tmdb - // msg.Payload: set, del, expire, etc. - p.publishEvent(msg.Channel, msg.Payload) - } - } -} - -// publishEvent publishes a keyspace event to the meta:events stream -func (p *MetaPublisher) publishEvent(channel, operation string) { - // Extract key from channel: __keyspace@0__:file:xxx/field -> file:xxx/field - key := strings.TrimPrefix(channel, "__keyspace@0__:") - - // Skip index key and non-file keys - if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") { - return - } - // Skip the per-record field index. It is bookkeeping (see - // storage.FieldsField), not a metadata field — publishing it would make - // every downstream consumer that parses `file:/` off this - // stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom - // field change on a key they can never read. - if strings.HasSuffix(key, "/"+storage.FieldsField) { - return - } - - // Build event fields - fields := map[string]interface{}{ - "type": operation, // "set", "del", "expire" - "key": key, // "file:midhash256:abc123/tmdb" - "ts": fmt.Sprintf("%d", time.Now().UnixMilli()), - } - - // Approximate MAXLEN trim keeps the stream bounded across restarts - // (see Start). Cap is intentionally generous; most consumers reconnect - // well within retention. - _, err := p.client.XAdd(p.ctx, &redis.XAddArgs{ - Stream: MetaEventsStream, - MaxLen: MetaEventsMaxLen, - Approx: true, - Values: fields, - }).Result() - - if err != nil { - log.Printf("[MetaPublisher] Error publishing event: %v", err) - } -} - -// publishEventDirect publishes an event directly (not from keyspace notification) -func (p *MetaPublisher) publishEventDirect(operation, key string) { - fields := map[string]interface{}{ - "type": operation, - "key": key, - "ts": fmt.Sprintf("%d", time.Now().UnixMilli()), - } - - _, err := p.client.XAdd(p.ctx, &redis.XAddArgs{ - Stream: MetaEventsStream, - MaxLen: MetaEventsMaxLen, - Approx: true, - Values: fields, - }).Result() - - if err != nil { - log.Printf("[MetaPublisher] Error publishing direct event: %v", err) - } -} - // RepublishFunc is a function type for getting metadata that can be republished type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error) -// RepublishAllMetadata iterates over all existing file metadata and publishes events -// This is called on startup and can be triggered via API +// RepublishAllMetadata publishes a `set` event for every property of every +// record. Called on startup and via the API. +// +// It exists for consumers that rebuild their state by replaying meta:events +// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather +// than backfilling over HTTP. Appended in bounded batches, each its own +// transaction, so a multi-million-event republish never holds the writer. func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) { - p.mu.RLock() - running := p.running - p.mu.RUnlock() - - if !running { + if !p.IsRunning() { return 0, fmt.Errorf("publisher not running") } - // Get all hash IDs and metadata getter hashIDs, getMetadata, err := republishFunc() if err != nil { return 0, fmt.Errorf("failed to get hash IDs: %w", err) @@ -201,22 +91,39 @@ func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs)) - publishedCount := 0 + published := 0 + batch := make([]map[string]string, 0, republishBatch) + flush := func() error { + if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil { + return err + } + published += len(batch) + batch = batch[:0] + return nil + } for _, hashID := range hashIDs { - // Get all properties for this file metadata, err := getMetadata(hashID) if err != nil || metadata == nil { continue } - - // Publish a set event for each property + ts := fmt.Sprintf("%d", time.Now().UnixMilli()) for property := range metadata { - key := fmt.Sprintf("file:%s/%s", hashID, property) - p.publishEventDirect("set", key) - publishedCount++ + batch = append(batch, map[string]string{ + "type": "set", + "key": fmt.Sprintf("file:%s/%s", hashID, property), + "ts": ts, + }) + if len(batch) >= republishBatch { + if err := flush(); err != nil { + return published, fmt.Errorf("republish: %w", err) + } + } } } + if err := flush(); err != nil { + return published, fmt.Errorf("republish: %w", err) + } - log.Printf("[MetaPublisher] Republished %d property events", publishedCount) - return publishedCount, nil + log.Printf("[MetaPublisher] Republished %d property events", published) + return published, nil } diff --git a/internal/identity/verify.go b/internal/identity/verify.go index e0bba30..7c7edc2 100644 --- a/internal/identity/verify.go +++ b/internal/identity/verify.go @@ -56,7 +56,7 @@ func ValidAction(a Action) bool { } // ValidUID exposes the uid shape check to callers outside this package (the -// API layer, which must reject a hostile uid before it reaches a Redis SCAN +// API layer, which must reject a hostile uid before it reaches a key-range read // pattern or a file path). func ValidUID(uid string) bool { return validUID(uid) } diff --git a/internal/kv/kv.go b/internal/kv/kv.go new file mode 100644 index 0000000..8d1c161 --- /dev/null +++ b/internal/kv/kv.go @@ -0,0 +1,713 @@ +// Package kv is meta-core's storage engine: one string→string table in an +// embedded SQLite file. +// +// # Why a key/value table and not a schema +// +// meta-core's data model is schema-free on purpose — a record is a set of flat +// `file:/` keys, and a new metadata field is just a new key +// (METADATA_KEYS.md). The engine keeps exactly that model: +// +// CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID +// +// is the only table that holds data, and it never needs a migration. What +// Redis gave us that a bare KV would not — "list a record's fields", "find +// records by field value", free-text search — comes from the table being +// *ordered* (a record is a key range) and from two derived structures the +// storage package maintains on top: an expression index over the field name +// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv +// rows and never authoritative. +// +// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a +// 169k-record corpus (Redis plus the API's in-memory copy needed to search +// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here +// and the working set is a bounded page cache. +// +// # Concurrency +// +// SQLite allows one writer at a time, so every write goes through one writer +// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED: +// the writer runs every request waiting at that moment inside one SQLite +// transaction, each under its own SAVEPOINT (a failing request rolls back +// alone), and commits once. A commit — writing the dirty pages of the table, +// the field index and the FTS shadow tables to the WAL — was ~40% of the cost +// of a record write, so sharing it is what makes concurrent ingest scale. +// +// Reads use a separate pool and see the last committed state (WAL). Inside +// Update, read through the *Tx — the reader pool cannot see uncommitted rows — +// and never call Update from inside an Update callback (the writer would wait +// for itself). +// +// # Events +// +// Redis keyspace notifications are replaced by the transaction itself: every +// Put/Delete of a key the Options.EventKey predicate selects is appended, in +// the same transaction, to Options.EventStream as {type: set|del, key, ts}. +// A change and its event therefore commit together or not at all. +package kv + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "net/url" + "os" + "strings" + "sync" + "time" + + _ "modernc.org/sqlite" +) + +// Options configures a Store. +type Options struct { + // EventStream receives a set/del entry for every changed key EventKey + // selects. Empty disables change events. + EventStream string + // EventMaxLen is the approximate retention of EventStream. + EventMaxLen int64 + // EventKey selects the keys whose changes are published. + EventKey func(key string) bool + // WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection. + WriterCacheMB int + ReaderCacheMB int + // Readers is the size of the read connection pool. + Readers int + // Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS), + // for derived indexes owned by the caller. + Schema string +} + +// Store is an open database. +type Store struct { + path string + opts Options + w *sql.DB // single connection: the one writer + r *sql.DB // read pool + + streamMu sync.Mutex + streams map[string]*streamState + + // Hot write statements, prepared once on the writer connection: parsing + // SQL on every Put was the single largest cost of a record write. + stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt + + writes chan *writeReq + writeMu sync.Mutex // held by the writer while a group runs + closeOnce sync.Once + stopped chan struct{} +} + +// writeReq is one Update call waiting for the writer. +type writeReq struct { + fn func(tx *Tx) error + done chan error +} + +// maxGroup caps how many requests share one commit, bounding how long the +// writer holds a transaction open. +const maxGroup = 128 + +// Open opens (creating if needed) the database file at path. +func Open(path string, opts Options) (*Store, error) { + if opts.Readers <= 0 { + opts.Readers = 8 + } + if opts.WriterCacheMB <= 0 { + opts.WriterCacheMB = 64 + } + if opts.ReaderCacheMB <= 0 { + opts.ReaderCacheMB = 16 + } + if opts.EventMaxLen <= 0 { + opts.EventMaxLen = 100_000 + } + + w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false)) + if err != nil { + return nil, fmt.Errorf("open writer: %w", err) + } + w.SetMaxOpenConns(1) + w.SetMaxIdleConns(1) + w.SetConnMaxLifetime(0) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil { + w.Close() + return nil, fmt.Errorf("create kv: %w", err) + } + if opts.Schema != "" { + // No deadline: on an existing store, the first open after a new index + // is added builds it over every row (minutes at millions of keys). + // Cancelling that just rolls it back and restarts it on the next boot. + started := time.Now() + if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil { + w.Close() + return nil, fmt.Errorf("apply schema: %w", err) + } + if d := time.Since(started); d > 5*time.Second { + log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second)) + } + } + + r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true)) + if err != nil { + w.Close() + return nil, fmt.Errorf("open readers: %w", err) + } + r.SetMaxOpenConns(opts.Readers) + r.SetMaxIdleConns(opts.Readers) + + st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)} + // Fresh deadline: the schema step above may legitimately have taken minutes. + ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, p := range []struct { + dst **sql.Stmt + query string + }{ + {&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`}, + {&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`}, + {&st.stDelete, `DELETE FROM kv WHERE k = ?`}, + {&st.stGet, `SELECT v FROM kv WHERE k = ?`}, + {&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`}, + } { + if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil { + r.Close() + w.Close() + return nil, fmt.Errorf("prepare: %w", err) + } + } + st.writes = make(chan *writeReq) + st.stopped = make(chan struct{}) + go st.writer() + return st, nil +} + +// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class +// Redis' AOF-everysec gave us: a power loss may drop the last transactions, +// never corrupt the file. +func dsn(path string, cacheMB int, readOnly bool) string { + q := url.Values{} + q.Add("_pragma", "busy_timeout(30000)") + q.Add("_pragma", "journal_mode(WAL)") + q.Add("_pragma", "synchronous(NORMAL)") + q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024)) + q.Add("_pragma", "temp_store(MEMORY)") + q.Add("_pragma", "journal_size_limit(268435456)") + if readOnly { + q.Add("_pragma", "query_only(1)") + } else { + q.Add("_txlock", "immediate") + } + return "file:" + path + "?" + q.Encode() +} + +// Close stops the writer and closes both pools, checkpointing the WAL into the +// main file. +func (s *Store) Close() error { + s.closeOnce.Do(func() { close(s.stopped) }) + s.writeMu.Lock() // wait for an in-flight group to finish + defer s.writeMu.Unlock() + _, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) + return errors.Join(s.r.Close(), s.w.Close()) +} + +// Ping reports whether the database answers. +func (s *Store) Ping(ctx context.Context) error { + var one int + return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one) +} + +// Path is the database file path. +func (s *Store) Path() string { return s.path } + +// DB exposes the read pool for callers that own derived indexes (search). It +// is read-only (query_only), so it cannot bypass Update. +func (s *Store) DB() *sql.DB { return s.r } + +// FileSize returns the on-disk size of the database plus its WAL. +func (s *Store) FileSize() int64 { + var total int64 + for _, suffix := range []string{"", "-wal"} { + if fi, err := os.Stat(s.path + suffix); err == nil { + total += fi.Size() + } + } + return total +} + +// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer +// slot and waits for readers, so callers run it when idle. +func (s *Store) Checkpoint(ctx context.Context) error { + _, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`) + return err +} + +// KV is one key/value pair. +type KV struct { + K, V string +} + +// Reader is the read surface shared by Store (committed state) and Tx (the +// transaction's own view). Storage helpers take a Reader so they work both +// inside and outside a write. +type Reader interface { + Get(ctx context.Context, k string) (string, bool, error) + MGet(ctx context.Context, keys []string) (map[string]string, error) + Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) + Count(ctx context.Context, prefix string) (int64, error) +} + +type queryer interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// PrefixEnd returns the smallest string greater than every string with the +// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range. +// An empty result means "unbounded". +func PrefixEnd(prefix string) string { + b := []byte(prefix) + for i := len(b) - 1; i >= 0; i-- { + if b[i] < 0xff { + b[i]++ + return string(b[:i+1]) + } + } + return "" +} + +func get(ctx context.Context, q queryer, k string) (string, bool, error) { + var v string + err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return v, true, nil +} + +// mgetBatch stays well under SQLite's default 32766 bound-parameter cap. +const mgetBatch = 500 + +func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) { + out := make(map[string]string, len(keys)) + for start := 0; start < len(keys); start += mgetBatch { + end := min(start+mgetBatch, len(keys)) + chunk := keys[start:end] + args := make([]any, len(chunk)) + for i, k := range chunk { + args[i] = k + } + rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...) + if err != nil { + return nil, err + } + for rows.Next() { + var k, v string + if err := rows.Scan(&k, &v); err != nil { + rows.Close() + return nil, err + } + out[k] = v + } + if err := rows.Close(); err != nil { + return nil, err + } + } + return out, nil +} + +// rangeQuery returns keys with prefix, strictly after `after` (when non-empty), +// in key order, at most limit (<= 0 means all). +func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) { + lo, loOp := prefix, ">=" + if after != "" && after >= prefix { + lo, loOp = after, ">" + } + query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?` + args := []any{lo} + if end := PrefixEnd(prefix); end != "" { + query += ` AND k < ?` + args = append(args, end) + } + query += ` ORDER BY k` + if limit > 0 { + query += ` LIMIT ?` + args = append(args, limit) + } + rows, err := q.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []KV + for rows.Next() { + var kv KV + if err := rows.Scan(&kv.K, &kv.V); err != nil { + return nil, err + } + out = append(out, kv) + } + return out, rows.Err() +} + +func count(ctx context.Context, q queryer, prefix string) (int64, error) { + query := `SELECT count(*) FROM kv WHERE k >= ?` + args := []any{prefix} + if end := PrefixEnd(prefix); end != "" { + query += ` AND k < ?` + args = append(args, end) + } + var n int64 + err := q.QueryRowContext(ctx, query, args...).Scan(&n) + return n, err +} + +// Get returns the committed value of k. +func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) } + +// MGet returns the committed values of the keys that exist. +func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) { + return mget(ctx, s.r, keys) +} + +// Range returns committed keys under prefix after `after`, in key order. +func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) { + return rangeQuery(ctx, s.r, prefix, after, limit) +} + +// Count returns how many committed keys sit under prefix. +func (s *Store) Count(ctx context.Context, prefix string) (int64, error) { + return count(ctx, s.r, prefix) +} + +// ScanPrefix walks every committed key under prefix in pages of `page`, each +// page its own short read so a long walk never pins a WAL snapshot (which +// would stop checkpoints and let the WAL grow without bound). +func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error { + if page <= 0 { + page = 5000 + } + after := "" + for { + batch, err := rangeQuery(ctx, s.r, prefix, after, page) + if err != nil { + return err + } + if len(batch) == 0 { + return nil + } + if err := fn(batch); err != nil { + return err + } + if len(batch) < page { + return nil + } + after = batch[len(batch)-1].K + } +} + +// CountAll counts every row. A full scan — callers cache it. +func (s *Store) CountAll(ctx context.Context) (int64, error) { + var n int64 + err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n) + return n, err +} + +// ErrClosed is returned by Update on a closed store. +var ErrClosed = errors.New("kv: store closed") + +// Update runs fn atomically: its writes (and the change events they publish) +// commit together or not at all. fn may share a SQLite transaction with other +// concurrent Updates (group commit) but is isolated from their failures by a +// savepoint. Never call Update from inside fn. +// +// ctx bounds only the wait for the writer. Once fn starts it runs to +// completion: interrupting a statement inside a shared transaction would roll +// back every request in the group. +func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error { + req := &writeReq{fn: fn, done: make(chan error, 1)} + select { + case s.writes <- req: + case <-ctx.Done(): + return ctx.Err() + case <-s.stopped: + return ErrClosed + } + return <-req.done +} + +// writer is the single goroutine that executes writes. +func (s *Store) writer() { + for { + var first *writeReq + select { + case first = <-s.writes: + case <-s.stopped: + return + } + group := []*writeReq{first} + drain: + for len(group) < maxGroup { + select { + case r := <-s.writes: + group = append(group, r) + default: + break drain + } + } + s.writeMu.Lock() + s.runGroup(group) + s.writeMu.Unlock() + } +} + +// group is the state shared by the requests of one commit. +type group struct { + tx *sql.Tx + pending map[string]*streamPending + stmts map[*sql.Stmt]*sql.Stmt +} + +func (g *group) snapshot() map[string]streamPending { + out := make(map[string]streamPending, len(g.pending)) + for k, p := range g.pending { + out[k] = *p + } + return out +} + +func (g *group) restore(snap map[string]streamPending) { + g.pending = make(map[string]*streamPending, len(snap)) + for k, p := range snap { + p := p + g.pending[k] = &p + } +} + +func (s *Store) runGroup(reqs []*writeReq) { + ctx := context.Background() + results := make([]error, len(reqs)) + fail := func(err error) { + for i, r := range reqs { + if results[i] == nil { + results[i] = err + } + r.done <- results[i] + } + } + + sqlTx, err := s.w.BeginTx(ctx, nil) + if err != nil { + fail(fmt.Errorf("begin: %w", err)) + return + } + g := &group{tx: sqlTx} + for i, r := range reqs { + if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil { + _ = sqlTx.Rollback() + fail(fmt.Errorf("savepoint: %w", err)) + return + } + snap := g.snapshot() + tx := &Tx{ctx: ctx, s: s, g: g} + err := r.fn(tx) + if err == nil { + err = tx.flushEvents() + } + if err != nil { + results[i] = err + if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil { + _ = sqlTx.Rollback() + fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err)) + return + } + g.restore(snap) + } + if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil { + _ = sqlTx.Rollback() + fail(fmt.Errorf("release savepoint: %w", err)) + return + } + } + if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil { + _ = sqlTx.Rollback() + fail(err) + return + } + if err := sqlTx.Commit(); err != nil { + fail(fmt.Errorf("commit: %w", err)) + return + } + s.afterCommit(g.pending) + for i, r := range reqs { + r.done <- results[i] + } +} + +// Tx is one Update's view of the write transaction. Its reads see its own +// writes (and those of earlier requests in the same group). +type Tx struct { + ctx context.Context + s *Store + g *group + events []map[string]string +} + +func (t *Tx) sqlTx() *sql.Tx { return t.g.tx } + +// stmt binds one of the store's prepared statements to the transaction. +func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt { + if t.g.stmts == nil { + t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5) + } + bound := t.g.stmts[st] + if bound == nil { + bound = t.g.tx.StmtContext(t.ctx, st) + t.g.stmts[st] = bound + } + return bound +} + +// Context is the context the transaction runs under. +func (t *Tx) Context() context.Context { return t.ctx } + +func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) { + var v string + err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, err + } + return v, true, nil +} +func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) { + return mget(ctx, t.sqlTx(), keys) +} +func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) { + return rangeQuery(ctx, t.sqlTx(), prefix, after, limit) +} +func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) { + return count(ctx, t.sqlTx(), prefix) +} + +// Exec runs a raw statement in the transaction — for the caller's derived +// tables (FTS), never for kv rows (which would skip events). +func (t *Tx) Exec(query string, args ...any) (sql.Result, error) { + return t.sqlTx().ExecContext(t.ctx, query, args...) +} + +// QueryRow runs a raw single-row query in the transaction. +func (t *Tx) QueryRow(query string, args ...any) *sql.Row { + return t.sqlTx().QueryRowContext(t.ctx, query, args...) +} + +// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis +// keyspace notifications did. +func (t *Tx) Put(k, v string) error { + if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil { + return fmt.Errorf("put %q: %w", k, err) + } + t.event("set", k) + return nil +} + +// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote. +// Publishes nothing when the key already existed (nothing changed). +func (t *Tx) PutIfAbsent(k, v string) (bool, error) { + res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v) + if err != nil { + return false, fmt.Errorf("put %q: %w", k, err) + } + n, _ := res.RowsAffected() + if n > 0 { + t.event("set", k) + } + return n > 0, nil +} + +// Delete removes k. Reports whether it existed; publishes `del` only then, +// as Redis did. +func (t *Tx) Delete(k string) (bool, error) { + res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k) + if err != nil { + return false, fmt.Errorf("delete %q: %w", k, err) + } + n, _ := res.RowsAffected() + if n > 0 { + t.event("del", k) + } + return n > 0, nil +} + +// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and +// returns how many went. Bulk callers loop over separate Updates with a limit +// so one transaction never grows to millions of rows. +func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) { + end := PrefixEnd(prefix) + sel := `SELECT k FROM kv WHERE k >= ?` + args := []any{prefix} + if end != "" { + sel += ` AND k < ?` + args = append(args, end) + } + if limit > 0 { + sel += ` ORDER BY k LIMIT ?` + args = append(args, limit) + } + rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...) + if err != nil { + return 0, fmt.Errorf("delete prefix %q: %w", prefix, err) + } + var keys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return 0, err + } + keys = append(keys, k) + } + if err := rows.Close(); err != nil { + return 0, err + } + for _, k := range keys { + t.event("del", k) + } + return int64(len(keys)), nil +} + +func (t *Tx) event(op, k string) { + if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) { + return + } + t.events = append(t.events, map[string]string{ + "type": op, + "key": k, + "ts": fmt.Sprintf("%d", time.Now().UnixMilli()), + }) +} + +// flushEvents appends the queued change events to the event stream, inside +// the transaction, right before commit. +func (t *Tx) flushEvents() error { + if len(t.events) == 0 { + return nil + } + evs := t.events + t.events = nil + for _, e := range evs { + if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil { + return err + } + } + return nil +} diff --git a/internal/kv/kv_test.go b/internal/kv/kv_test.go new file mode 100644 index 0000000..865d1c4 --- /dev/null +++ b/internal/kv/kv_test.go @@ -0,0 +1,314 @@ +package kv + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" +) + +func openTest(t *testing.T, path string) *Store { + t.Helper() + if path == "" { + path = filepath.Join(t.TempDir(), "kv.db") + } + s, err := Open(path, Options{ + EventStream: "meta:events", + EventMaxLen: 100_000, + EventKey: func(k string) bool { return strings.HasPrefix(k, "file:") }, + }) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func update(t *testing.T, s *Store, fn func(tx *Tx) error) { + t.Helper() + if err := s.Update(context.Background(), fn); err != nil { + t.Fatalf("update: %v", err) + } +} + +func events(t *testing.T, s *Store) []Entry { + t.Helper() + es, err := s.ReadAfter(context.Background(), "meta:events", "0", 0) + if err != nil { + t.Fatalf("read events: %v", err) + } + return es +} + +// The replacement for keyspace notifications: a change and its event commit +// together, with Redis' semantics — `set` even for an unchanged value, `del` +// only for a key that existed, and nothing for keys outside the event scope. +func TestChangesPublishEventsWithRedisSemantics(t *testing.T) { + s := openTest(t, "") + update(t, s, func(tx *Tx) error { + _ = tx.Put("file:A/title", "x") + _ = tx.Put("file:A/title", "x") // unchanged value still publishes + _ = tx.Put("cid:bafk", "A") // outside the event scope + _, _ = tx.Delete("file:A/missing") + _, _ = tx.Delete("file:A/title") + return nil + }) + got := events(t, s) + want := []string{"set file:A/title", "set file:A/title", "del file:A/title"} + if len(got) != len(want) { + t.Fatalf("events = %+v, want %v", got, want) + } + for i, e := range got { + if e.Values["type"]+" "+e.Values["key"] != want[i] { + t.Errorf("event %d = %v, want %q", i, e.Values, want[i]) + } + } +} + +func TestRolledBackChangesPublishNothing(t *testing.T) { + s := openTest(t, "") + err := s.Update(context.Background(), func(tx *Tx) error { + _ = tx.Put("file:A/title", "x") + return errors.New("boom") + }) + if err == nil { + t.Fatal("want the error back") + } + if _, ok, _ := s.Get(context.Background(), "file:A/title"); ok { + t.Fatal("rolled-back write is visible") + } + if es := events(t, s); len(es) != 0 { + t.Fatalf("rolled-back write published %+v", es) + } +} + +func TestDeletePrefixPublishesOneDelPerKey(t *testing.T) { + s := openTest(t, "") + update(t, s, func(tx *Tx) error { + for _, k := range []string{"file:A/a", "file:A/b", "file:AB/c"} { + _ = tx.Put(k, "1") + } + return nil + }) + var n int64 + update(t, s, func(tx *Tx) error { + var err error + n, err = tx.DeletePrefix("file:A/", 0) + return err + }) + if n != 2 { + t.Fatalf("deleted %d, want 2 (file:AB/ is not under file:A/)", n) + } + es := events(t, s) + if dels := es[3:]; len(dels) != 2 || dels[0].Values["type"] != "del" || dels[1].Values["type"] != "del" { + t.Fatalf("want two del events, got %+v", es) + } +} + +// IDs are Redis-shaped and strictly increasing, within one millisecond and +// across a reopen — the SSE cursors consumers persist depend on both. +func TestStreamIDsAreMonotonicAcrossReopen(t *testing.T) { + path := filepath.Join(t.TempDir(), "kv.db") + s, err := Open(path, Options{}) + if err != nil { + t.Fatal(err) + } + var ids []string + if err := s.Update(context.Background(), func(tx *Tx) error { + for i := 0; i < 50; i++ { + id, err := tx.Append("file:events", 0, map[string]string{"n": "x"}) + if err != nil { + return err + } + ids = append(ids, id) + } + return nil + }); err != nil { + t.Fatal(err) + } + _ = s.Close() + + s2 := openTest(t, path) + var after string + update(t, s2, func(tx *Tx) error { + var err error + after, err = tx.Append("file:events", 0, map[string]string{"n": "y"}) + return err + }) + ids = append(ids, after) + for i := 1; i < len(ids); i++ { + if !idLess(t, ids[i-1], ids[i]) { + t.Fatalf("id %d (%s) does not follow %s", i, ids[i], ids[i-1]) + } + } + if !strings.Contains(ids[0], "-") || strings.HasPrefix(ids[0], "0") { + t.Fatalf("id %q is not the unpadded - form", ids[0]) + } +} + +func idLess(t *testing.T, a, b string) bool { + t.Helper() + am, as, err := ParseID(a) + if err != nil { + t.Fatal(err) + } + bm, bs, err := ParseID(b) + if err != nil { + t.Fatal(err) + } + return am < bm || (am == bm && as < bs) +} + +// The watcher's reset clears file:events. The next ID must still exceed every +// ID issued before the clear, or a consumer holding an old cursor would skip +// the reset event. +func TestClearStreamKeepsTheHighWaterMark(t *testing.T) { + s := openTest(t, "") + var before, after string + update(t, s, func(tx *Tx) error { + var err error + for i := 0; i < 5; i++ { + if before, err = tx.Append("file:events", 0, map[string]string{"type": "add"}); err != nil { + return err + } + } + return nil + }) + update(t, s, func(tx *Tx) error { return tx.ClearStream("file:events") }) + if n, _ := s.Len(context.Background(), "file:events"); n != 0 { + t.Fatalf("len after clear = %d", n) + } + update(t, s, func(tx *Tx) error { + var err error + after, err = tx.Append("file:events", 0, map[string]string{"type": "reset"}) + return err + }) + if !idLess(t, before, after) { + t.Fatalf("post-clear id %s does not follow pre-clear %s", after, before) + } + got, err := s.ReadAfter(context.Background(), "file:events", before, 10) + if err != nil || len(got) != 1 || got[0].Values["type"] != "reset" { + t.Fatalf("an old cursor must see the reset: %+v err=%v", got, err) + } +} + +// MAXLEN ~ semantics across many small transactions (the common case: one +// event per write). +func TestStreamTrimsAcrossTransactions(t *testing.T) { + defer func(old int64) { trimEvery = old }(trimEvery) + trimEvery = 100 + s := openTest(t, "") + const maxLen = 50 + for i := int64(0); i < 3*trimEvery; i++ { + update(t, s, func(tx *Tx) error { + _, err := tx.Append("x", maxLen, map[string]string{"i": "1"}) + return err + }) + } + n, err := s.Len(context.Background(), "x") + if err != nil { + t.Fatal(err) + } + if n > maxLen+trimEvery { + t.Fatalf("stream holds %d entries, want <= %d", n, maxLen+trimEvery) + } + oldest, ok, err := s.Oldest(context.Background(), "x") + if err != nil || !ok { + t.Fatal(err) + } + if last, _ := s.LastID(context.Background(), "x"); !idLess(t, oldest.ID, last) { + t.Fatalf("oldest %s should precede last %s", oldest.ID, last) + } +} + +// A subscriber that took the wake channel before reading is woken by the next +// commit — the property the SSE pump relies on to not miss an append. +func TestWaitIsClosedByTheNextCommit(t *testing.T) { + s := openTest(t, "") + wake := s.Wait("meta:events") + update(t, s, func(tx *Tx) error { return tx.Put("file:A/x", "1") }) + select { + case <-wake: + case <-time.After(2 * time.Second): + t.Fatal("commit did not wake the waiter") + } +} + +func TestReadAfterZeroFormsReadFromTheStart(t *testing.T) { + s := openTest(t, "") + update(t, s, func(tx *Tx) error { + _, err := tx.Append("x", 0, map[string]string{"n": "1"}) + return err + }) + for _, id := range []string{"", "0", "0-0"} { + got, err := s.ReadAfter(context.Background(), "x", id, 10) + if err != nil || len(got) != 1 { + t.Errorf("ReadAfter(%q) = %d entries, err %v; want 1", id, len(got), err) + } + } +} + +func TestRangeIsExactlyThePrefix(t *testing.T) { + s := openTest(t, "") + update(t, s, func(tx *Tx) error { + for _, k := range []string{"file:A/a", "file:A/b/c", "file:A0", "file:AB/x", "file:A"} { + _ = tx.Put(k, "v") + } + return nil + }) + rows, err := s.Range(context.Background(), "file:A/", "", 0) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 || rows[0].K != "file:A/a" || rows[1].K != "file:A/b/c" { + t.Fatalf("range = %+v", rows) + } +} + +// Concurrent Updates share commits; a failing one must roll back alone and +// leave its neighbours' writes and events intact. +func TestGroupCommitIsolatesFailures(t *testing.T) { + s := openTest(t, "") + const n = 200 + errs := make(chan error, n) + for i := 0; i < n; i++ { + go func(i int) { + errs <- s.Update(context.Background(), func(tx *Tx) error { + k := fmt.Sprintf("file:R%03d/title", i) + if err := tx.Put(k, "x"); err != nil { + return err + } + if i%10 == 0 { + return errors.New("fail") + } + return nil + }) + }(i) + } + failed := 0 + for i := 0; i < n; i++ { + if <-errs != nil { + failed++ + } + } + if failed != n/10 { + t.Fatalf("failed = %d, want %d", failed, n/10) + } + rows, _ := s.Range(context.Background(), "file:", "", 0) + if len(rows) != n-n/10 { + t.Fatalf("committed %d rows, want %d", len(rows), n-n/10) + } + if es := events(t, s); len(es) != n-n/10 { + t.Fatalf("events = %d, want %d (one per committed write, none for failures)", len(es), n-n/10) + } + for _, r := range rows { + var i int + fmt.Sscanf(r.K, "file:R%03d/title", &i) + if i%10 == 0 { + t.Fatalf("failed request's write %s committed", r.K) + } + } +} diff --git a/internal/kv/stream.go b/internal/kv/stream.go new file mode 100644 index 0000000..880839b --- /dev/null +++ b/internal/kv/stream.go @@ -0,0 +1,356 @@ +package kv + +// Append-only event streams, as rows of the kv table. +// +// A stream entry is the row +// +// ev:/- → JSON object of the entry's string fields +// +// Zero padding makes key order equal ID order, so reading "after an ID" is a +// plain range read and trimming is a range delete. On the wire IDs are the +// unpadded Redis form (`1700000000000-0`): meta-sort, meta-share and +// meta-stremio persist SSE cursors to disk across the Redis → SQLite switch, +// and those compare as - pairs. +// +// The last issued ID is kept in `evhwm:` and updated in the same +// transaction as the append. That high-water mark, not the newest surviving +// row, seeds the next ID — so Clear (the watcher's `reset`) never reissues an +// ID an old cursor would treat as already seen. + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +// Entry is one stream entry. +type Entry struct { + ID string + Values map[string]string +} + +type streamID struct{ ms, seq int64 } + +func (id streamID) String() string { + return strconv.FormatInt(id.ms, 10) + "-" + strconv.FormatInt(id.seq, 10) +} + +func (id streamID) padded() string { return fmt.Sprintf("%013d-%010d", id.ms, id.seq) } + +func (id streamID) less(o streamID) bool { + return id.ms < o.ms || (id.ms == o.ms && id.seq < o.seq) +} + +// ParseID parses a Redis-style stream ID. "", "0", "0-0", "-" all parse as the +// zero ID (before everything). A bare "" means "-0". +func ParseID(s string) (ms, seq int64, err error) { + if s == "" || s == "-" || s == "0" { + return 0, 0, nil + } + msPart, seqPart, hasSeq := strings.Cut(s, "-") + ms, err = strconv.ParseInt(msPart, 10, 64) + if err != nil || ms < 0 { + return 0, 0, fmt.Errorf("invalid stream id %q", s) + } + if hasSeq { + seq, err = strconv.ParseInt(seqPart, 10, 64) + if err != nil || seq < 0 { + return 0, 0, fmt.Errorf("invalid stream id %q", s) + } + } + return ms, seq, nil +} + +func streamPrefix(stream string) string { return "ev:" + stream + "/" } +func hwmKey(stream string) string { return "evhwm:" + stream } + +// streamState is the in-memory view of a stream's high-water mark plus the +// broadcast channel subscribers wait on. Updated only after a commit. +type streamState struct { + loaded bool + hwm streamID + sinceTrim int64 // committed appends since the last trim + notify chan struct{} +} + +// streamPending is what a transaction did to one stream, applied to +// streamState on commit (and discarded on rollback). +type streamPending struct { + hwm streamID + appends int64 + trimmed bool + lastTrimAt int64 // value of appends when this transaction last trimmed +} + +func (s *Store) state(stream string) *streamState { + st := s.streams[stream] + if st == nil { + st = &streamState{notify: make(chan struct{})} + s.streams[stream] = st + } + return st +} + +// Wait returns a channel closed by the next commit that appends to stream. +// Take it BEFORE reading, so an append between the read and the wait is not +// missed. +func (s *Store) Wait(stream string) <-chan struct{} { + s.streamMu.Lock() + defer s.streamMu.Unlock() + return s.state(stream).notify +} + +func (s *Store) afterCommit(pending map[string]*streamPending) { + if len(pending) == 0 { + return + } + s.streamMu.Lock() + defer s.streamMu.Unlock() + for name, p := range pending { + st := s.state(name) + st.loaded = true + st.hwm = p.hwm + if p.trimmed { + st.sinceTrim = p.appends - p.lastTrimAt + } else { + st.sinceTrim += p.appends + } + close(st.notify) + st.notify = make(chan struct{}) + } +} + +// hwm returns the stream's last issued ID as seen by this transaction. +func (t *Tx) hwm(stream string) (streamID, error) { + if p := t.g.pending[stream]; p != nil { + return p.hwm, nil + } + t.s.streamMu.Lock() + st := t.s.state(stream) + loaded, hwm := st.loaded, st.hwm + t.s.streamMu.Unlock() + if loaded { + return hwm, nil + } + v, ok, err := t.Get(t.ctx, hwmKey(stream)) + if err != nil { + return streamID{}, err + } + if !ok { + return streamID{}, nil + } + ms, seq, err := ParseID(v) + if err != nil { + return streamID{}, err + } + return streamID{ms, seq}, nil +} + +// Append adds an entry to stream and returns its ID. The stream is trimmed to +// about maxLen entries (maxLen <= 0: unbounded). +func (t *Tx) Append(stream string, maxLen int64, fields map[string]string) (string, error) { + last, err := t.hwm(stream) + if err != nil { + return "", fmt.Errorf("stream %s hwm: %w", stream, err) + } + id := streamID{ms: time.Now().UnixMilli()} + if !last.less(id) { + id = streamID{ms: last.ms, seq: last.seq + 1} + } + body, err := json.Marshal(fields) + if err != nil { + return "", err + } + if _, err := t.stmt(t.s.stInsert).ExecContext(t.ctx, streamPrefix(stream)+id.padded(), string(body)); err != nil { + return "", fmt.Errorf("append %s: %w", stream, err) + } + if t.g.pending == nil { + t.g.pending = make(map[string]*streamPending) + } + p := t.g.pending[stream] + if p == nil { + p = &streamPending{} + t.g.pending[stream] = p + } + p.hwm = id + p.appends++ + // Trim in steps, not per append: MAXLEN ~ semantics, like the XADD it + // replaces. The count spans transactions (most append one event each). + if maxLen > 0 { + t.s.streamMu.Lock() + since := t.s.state(stream).sinceTrim + t.s.streamMu.Unlock() + if !p.trimmed { + since += p.appends + } else { + since = p.appends - p.lastTrimAt + } + if since >= trimEvery { + if err := t.trim(stream, maxLen); err != nil { + return "", err + } + p.trimmed, p.lastTrimAt = true, p.appends + } + } + return id.String(), nil +} + +// trimEvery is how many appends pass between trims. A trim locates its cut +// point by walking maxLen entries back from the newest, so it is kept rare; +// the cost is overshooting maxLen by at most trimEvery entries. +var trimEvery int64 = 10_000 // var so tests can lower it + +func (t *Tx) trim(stream string, maxLen int64) error { + prefix := streamPrefix(stream) + var cut string + err := t.sqlTx().QueryRowContext(t.ctx, + `SELECT k FROM kv WHERE k >= ? AND k < ? ORDER BY k DESC LIMIT 1 OFFSET ?`, + prefix, PrefixEnd(prefix), maxLen).Scan(&cut) + if err != nil { + return nil // fewer than maxLen entries (sql.ErrNoRows): nothing to trim + } + _, err = t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k <= ?`, prefix, cut) + return err +} + +// flushHWM persists each touched stream's high-water mark — once per +// transaction, not per append. +func (t *Tx) flushHWM() error { + for stream, p := range t.g.pending { + if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, hwmKey(stream), p.hwm.String()); err != nil { + return fmt.Errorf("stream %s hwm: %w", stream, err) + } + } + return nil +} + +// TrimAll trims stream to maxLen inside the transaction regardless of how many +// appends this transaction made. Used after an append burst split over many +// short transactions. +func (t *Tx) TrimAll(stream string, maxLen int64) error { return t.trim(stream, maxLen) } + +// ClearStream deletes every entry of stream, keeping its high-water mark. +func (t *Tx) ClearStream(stream string) error { + prefix := streamPrefix(stream) + _, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k < ?`, prefix, PrefixEnd(prefix)) + return err +} + +func decodeEntry(stream, k, v string) (Entry, error) { + padded := strings.TrimPrefix(k, streamPrefix(stream)) + ms, seq, err := ParseID(padded) + if err != nil { + return Entry{}, err + } + var values map[string]string + if err := json.Unmarshal([]byte(v), &values); err != nil { + return Entry{}, fmt.Errorf("decode %s: %w", k, err) + } + return Entry{ID: streamID{ms, seq}.String(), Values: values}, nil +} + +func decodeEntries(stream string, rows []KV) ([]Entry, error) { + out := make([]Entry, 0, len(rows)) + for _, kv := range rows { + e, err := decodeEntry(stream, kv.K, kv.V) + if err != nil { + return nil, err + } + out = append(out, e) + } + return out, nil +} + +// ReadAfter returns up to n entries strictly after id, oldest first. "0", +// "0-0" and "" read from the beginning. +func (s *Store) ReadAfter(ctx context.Context, stream, id string, n int) ([]Entry, error) { + ms, seq, err := ParseID(id) + if err != nil { + return nil, err + } + prefix := streamPrefix(stream) + after := "" + if ms != 0 || seq != 0 { + after = prefix + streamID{ms, seq}.padded() + } + rows, err := rangeQuery(ctx, s.r, prefix, after, n) + if err != nil { + return nil, err + } + return decodeEntries(stream, rows) +} + +// Oldest returns the oldest surviving entry, or ok=false for an empty stream. +func (s *Store) Oldest(ctx context.Context, stream string) (Entry, bool, error) { + rows, err := rangeQuery(ctx, s.r, streamPrefix(stream), "", 1) + if err != nil || len(rows) == 0 { + return Entry{}, false, err + } + e, err := decodeEntry(stream, rows[0].K, rows[0].V) + return e, err == nil, err +} + +// LastID returns the last ID ever issued on stream ("0-0" if none) — the +// cursor meaning "only entries appended from now on". +func (s *Store) LastID(ctx context.Context, stream string) (string, error) { + s.streamMu.Lock() + st := s.state(stream) + loaded, hwm := st.loaded, st.hwm + s.streamMu.Unlock() + if loaded { + return hwm.String(), nil + } + v, ok, err := s.Get(ctx, hwmKey(stream)) + if err != nil { + return "", err + } + if !ok { + return "0-0", nil + } + return v, nil +} + +// Len is the number of surviving entries. +func (s *Store) Len(ctx context.Context, stream string) (int64, error) { + return count(ctx, s.r, streamPrefix(stream)) +} + +// RevRange returns up to n entries newest first, stopping at (excluding) +// entries at or before afterID when afterID is non-empty. +func (s *Store) RevRange(ctx context.Context, stream, afterID string, n int) ([]Entry, error) { + prefix := streamPrefix(stream) + query := `SELECT k, v FROM kv WHERE k > ? AND k < ? ORDER BY k DESC` + lo := prefix + if afterID != "" { + ms, seq, err := ParseID(afterID) + if err != nil { + return nil, err + } + lo = prefix + streamID{ms, seq}.padded() + } + args := []any{lo, PrefixEnd(prefix)} + if n > 0 { + query += ` LIMIT ?` + args = append(args, n) + } + rows, err := s.r.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var kvs []KV + for rows.Next() { + var kv KV + if err := rows.Scan(&kv.K, &kv.V); err != nil { + return nil, err + } + kvs = append(kvs, kv) + } + if err := rows.Err(); err != nil { + return nil, err + } + return decodeEntries(stream, kvs) +} diff --git a/internal/leader/election.go b/internal/leader/election.go index 774322d..40c595c 100644 --- a/internal/leader/election.go +++ b/internal/leader/election.go @@ -13,7 +13,7 @@ import ( type LeaderLockInfo struct { Hostname string `json:"hostname"` BaseUrl string `json:"baseUrl"` - ApiUrl string `json:"apiUrl"` // meta-core API (port 9000) + ApiUrl string `json:"apiUrl"` // meta-core API (port 9000) RedisUrl string `json:"redisUrl"` WebdavUrl string `json:"webdavUrl"` // External WebDAV URL (via baseUrl) WebdavUrlInternal string `json:"webdavUrlInternal"` // Internal WebDAV URL (direct to port 9000) @@ -72,9 +72,9 @@ func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo { webdavUrlInternal := fmt.Sprintf("http://%s:%d/webdav", hostname, p.config.HTTPPort) return &LeaderLockInfo{ - Hostname: hostname, - BaseUrl: baseUrl, - ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort), + Hostname: hostname, + BaseUrl: baseUrl, + ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort), // RedisUrl deliberately empty — api-mediated-access PR D removes // direct Redis exposure. Consumers route metadata I/O through // meta-core's HTTP API (/meta/{hash}*) and event streams through diff --git a/internal/schema/classify.go b/internal/schema/classify.go index 5569ccb..5896dc1 100644 --- a/internal/schema/classify.go +++ b/internal/schema/classify.go @@ -21,7 +21,7 @@ import ( // (crc32: "bafk3eaqerjhiwhy"). var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`) -// ClassifyValue returns the (primitive, hint) for a Redis value. +// ClassifyValue returns the (primitive, hint) for a stored value. // exists=false (key missing) and JSON null literal both map to PrimUndefined. // Empty string is a valid string. func ClassifyValue(value string, exists bool) (Primitive, Hint) { diff --git a/internal/schema/indexer.go b/internal/schema/indexer.go index c2732d7..44afbef 100644 --- a/internal/schema/indexer.go +++ b/internal/schema/indexer.go @@ -6,17 +6,20 @@ import ( "regexp" "strings" "sync" - "sync/atomic" "time" - "github.com/metazla/meta-core/internal/events" - "github.com/redis/go-redis/v9" + "github.com/metazla/meta-core/internal/kv" ) var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`) -// fieldState is the per-field accumulator. perKey lets us undo a previous -// contribution when the same Redis key is re-SET or DELed. +// rebuildInterval is how often the schema is recomputed in the background. +// It describes the *shape* of the data (which fields exist, their types), which +// drifts slowly; a few minutes of staleness is invisible to the editor that +// reads it, and POST /api/schema/rescan recomputes on demand. +const rebuildInterval = 10 * time.Minute + +// fieldState is the per-field accumulator. type fieldState struct { counts map[string]int primitives map[Primitive]int @@ -24,12 +27,6 @@ type fieldState struct { typedCount int undefinedCount int languageKeyed bool - perKey map[string]labelPair -} - -type labelPair struct { - prim Primitive - hint Hint } func newFieldState() *fieldState { @@ -37,19 +34,25 @@ func newFieldState() *fieldState { counts: make(map[string]int), primitives: make(map[Primitive]int), hintCounts: make(map[Hint]int), - perKey: make(map[string]labelPair), } } -// Indexer maintains a live field-schema by consuming meta:events. +// Indexer maintains the per-field schema by scanning the store. +// +// It used to follow meta:events and keep, per field, a map of EVERY record key +// it had seen (so a re-set or delete could undo that key's contribution). At +// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident +// memory once the search copy was gone — and it only worked if every event was +// replayed into it, which is half of why meta-core republished every field on +// boot. A periodic streaming scan holds one accumulator per FIELD instead and +// is exact by construction. type Indexer struct { - client *redis.Client - keyPrefix string + store *kv.Store - mu sync.RWMutex - fields map[string]*fieldState + mu sync.RWMutex + snap *SchemaResponse - processed uint64 // total events processed since start + rebuildMu sync.Mutex ctx context.Context cancel context.CancelFunc @@ -58,37 +61,41 @@ type Indexer struct { running bool } -// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix() -// so the indexer can build absolute Redis keys for GET calls. -func NewIndexer(client *redis.Client, keyPrefix string) *Indexer { +// NewIndexer creates an Indexer over store. +func NewIndexer(store *kv.Store) *Indexer { ctx, cancel := context.WithCancel(context.Background()) - return &Indexer{ - client: client, - keyPrefix: keyPrefix, - fields: make(map[string]*fieldState), - ctx: ctx, - cancel: cancel, - } + return &Indexer{store: store, ctx: ctx, cancel: cancel} } -// Start begins consuming the meta:events stream from the earliest available ID. +// Start computes the schema in the background now and every rebuildInterval. func (i *Indexer) Start() error { i.runMu.Lock() + defer i.runMu.Unlock() if i.running { - i.runMu.Unlock() return nil } i.running = true - i.runMu.Unlock() - i.wg.Add(1) - go i.consumeLoop() - - log.Println("[SchemaIndexer] Started - consuming meta:events stream") + go func() { + defer i.wg.Done() + ticker := time.NewTicker(rebuildInterval) + defer ticker.Stop() + for { + if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil { + log.Printf("[SchemaIndexer] rebuild failed: %v", err) + } + select { + case <-i.ctx.Done(): + return + case <-ticker.C: + } + } + }() + log.Println("[SchemaIndexer] Started - scanning the store") return nil } -// Stop terminates the consumer goroutine. +// Stop terminates the background rebuilds. func (i *Indexer) Stop() error { i.runMu.Lock() if !i.running { @@ -104,93 +111,55 @@ func (i *Indexer) Stop() error { return nil } -// ProcessedCount returns the total number of stream events processed. -// Used by WaitForDrain to detect when a rescan flood has been absorbed. -func (i *Indexer) ProcessedCount() uint64 { - return atomic.LoadUint64(&i.processed) -} +// Rebuild recomputes the schema from every record key and publishes it. One +// rebuild runs at a time; a concurrent caller waits and gets the fresh result. +func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) { + i.rebuildMu.Lock() + defer i.rebuildMu.Unlock() -// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or -// the timeout elapses. Suitable for use right after RepublishMetadata. -func (i *Indexer) WaitForDrain(timeout time.Duration) { - deadline := time.Now().Add(timeout) - last := i.ProcessedCount() - stableSince := time.Now() - for time.Now().Before(deadline) { - time.Sleep(100 * time.Millisecond) - now := i.ProcessedCount() - if now != last { - last = now - stableSince = time.Now() - continue - } - if time.Since(stableSince) >= 300*time.Millisecond { - return - } - } -} - -func (i *Indexer) consumeLoop() { - defer i.wg.Done() - lastID := "0" - for { - select { - case <-i.ctx.Done(): - return - default: - } - - res, err := i.client.XRead(i.ctx, &redis.XReadArgs{ - Streams: []string{events.MetaEventsStream, lastID}, - Block: 2 * time.Second, - Count: 500, - }).Result() - - if err != nil { - if err == redis.Nil || i.ctx.Err() != nil { + started := time.Now() + fields := make(map[string]*fieldState) + err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error { + for _, row := range rows { + fieldPath, keyHint, ok := parseFieldPath(row.K) + if !ok { continue } - log.Printf("[SchemaIndexer] XRead error: %v", err) - time.Sleep(1 * time.Second) - continue - } - - for _, stream := range res { - for _, msg := range stream.Messages { - lastID = msg.ID - i.processEvent(msg.Values) - atomic.AddUint64(&i.processed, 1) + fs := fields[fieldPath] + if fs == nil { + fs = newFieldState() + fields[fieldPath] = fs } + if keyHint != "" { + fs.languageKeyed = true + } + prim, hint := ClassifyValue(row.V, true) + incContribution(fs, labelPair{prim: prim, hint: hint}) } + return ctx.Err() + }) + if err != nil { + return nil, err } + + snap := &SchemaResponse{ + Fields: make(map[string]*FieldSchema, len(fields)), + GeneratedAt: time.Now().UTC(), + Source: "scan", + } + for fp, fs := range fields { + snap.Fields[fp] = computeFieldSchema(fs) + } + i.mu.Lock() + i.snap = snap + i.mu.Unlock() + log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond)) + return snap, nil } -func (i *Indexer) processEvent(values map[string]any) { - op, _ := values["type"].(string) - key, _ := values["key"].(string) - if key == "" { - return - } - fieldPath, keyHint, ok := parseFieldPath(key) - if !ok { - return - } - - switch op { - case "set": - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - val, err := i.client.Get(ctx, i.keyPrefix+key).Result() - cancel() - exists := err != redis.Nil - if err != nil && err != redis.Nil { - // Treat transient errors as "value disappeared" — nothing to record. - return - } - prim, hint := ClassifyValue(val, exists) - i.upsert(fieldPath, key, keyHint, prim, hint) - case "del", "expire": - i.remove(fieldPath, key) - } +type labelPair struct { + prim Primitive + hint Hint } // parseFieldPath turns "file:/" into the logical field @@ -222,46 +191,6 @@ func parseFieldPath(key string) (string, KeyHint, bool) { return property, "", true } -func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) { - i.mu.Lock() - defer i.mu.Unlock() - - fs, ok := i.fields[fieldPath] - if !ok { - fs = newFieldState() - i.fields[fieldPath] = fs - } - if keyHint != "" { - fs.languageKeyed = true - } - - if old, ok := fs.perKey[redisKey]; ok { - decContribution(fs, old) - } - - fs.perKey[redisKey] = labelPair{prim: prim, hint: hint} - incContribution(fs, labelPair{prim: prim, hint: hint}) -} - -func (i *Indexer) remove(fieldPath, redisKey string) { - i.mu.Lock() - defer i.mu.Unlock() - - fs, ok := i.fields[fieldPath] - if !ok { - return - } - old, ok := fs.perKey[redisKey] - if !ok { - return - } - delete(fs.perKey, redisKey) - decContribution(fs, old) - if len(fs.perKey) == 0 { - delete(i.fields, fieldPath) - } -} - func incContribution(fs *fieldState, lp labelPair) { label := breakdownLabel(lp.prim, lp.hint) fs.counts[label]++ @@ -276,43 +205,15 @@ func incContribution(fs *fieldState, lp labelPair) { } } -func decContribution(fs *fieldState, lp labelPair) { - label := breakdownLabel(lp.prim, lp.hint) - fs.counts[label]-- - if fs.counts[label] <= 0 { - delete(fs.counts, label) - } - if lp.prim == PrimUndefined { - fs.undefinedCount-- - return - } - fs.typedCount-- - fs.primitives[lp.prim]-- - if fs.primitives[lp.prim] <= 0 { - delete(fs.primitives, lp.prim) - } - if lp.hint != "" { - fs.hintCounts[lp.hint]-- - if fs.hintCounts[lp.hint] <= 0 { - delete(fs.hintCounts, lp.hint) - } - } -} - -// Snapshot returns the current schema. Safe to call concurrently with consumption. +// Snapshot returns the latest computed schema. Before the first scan finishes +// it is empty with Source "pending". func (i *Indexer) Snapshot() *SchemaResponse { i.mu.RLock() defer i.mu.RUnlock() - - out := &SchemaResponse{ - Fields: make(map[string]*FieldSchema, len(i.fields)), - GeneratedAt: time.Now().UTC(), - Source: "live", + if i.snap == nil { + return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"} } - for fp, fs := range i.fields { - out.Fields[fp] = computeFieldSchema(fs) - } - return out + return i.snap } func computeFieldSchema(fs *fieldState) *FieldSchema { diff --git a/internal/schema/indexer_test.go b/internal/schema/indexer_test.go new file mode 100644 index 0000000..d1e5c7b --- /dev/null +++ b/internal/schema/indexer_test.go @@ -0,0 +1,56 @@ +package schema + +import ( + "context" + "path/filepath" + "testing" + + "github.com/metazla/meta-core/internal/kv" +) + +func TestRebuildDerivesTypesFromTheStore(t *testing.T) { + store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{}) + if err != nil { + t.Fatal(err) + } + defer store.Close() + if err := store.Update(context.Background(), func(tx *kv.Tx) error { + for k, v := range map[string]string{ + "file:A/sizeByte": "100", + "file:B/sizeByte": "200", + "file:A/title/eng": "One", + "file:B/title/fre": "Deux", + "file:A/mixed": "1", + "file:B/mixed": "x", + "cid:bafknotrecord": "A", + } { + if err := tx.Put(k, v); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatal(err) + } + + ix := NewIndexer(store) + if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 { + t.Fatalf("before a scan: %+v", got) + } + snap, err := ix.Rebuild(context.Background()) + if err != nil { + t.Fatal(err) + } + if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 { + t.Fatalf("sizeByte = %+v", f) + } + if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode { + t.Fatalf("title = %+v (want language-keyed)", f) + } + if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed { + t.Fatalf("mixed = %+v", f) + } + if len(snap.Fields) != 3 { + t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields) + } +} diff --git a/internal/snapshot/export.go b/internal/snapshot/export.go index bdf2d01..9e16ea6 100644 --- a/internal/snapshot/export.go +++ b/internal/snapshot/export.go @@ -24,7 +24,7 @@ type ExportOptions struct { SourceHost string } -// Exporter writes snapshot ZIPs from a live Redis store. +// Exporter writes snapshot ZIPs from the live metadata store. type Exporter struct { storage *storage.Client } diff --git a/internal/snapshot/flatten.go b/internal/snapshot/flatten.go index 603ed80..8758967 100644 --- a/internal/snapshot/flatten.go +++ b/internal/snapshot/flatten.go @@ -8,7 +8,7 @@ import ( "strings" ) -// Flat keys use "/" as the separator. Property paths inside Redis are +// Flat keys use "/" as the separator. Property paths inside the store are // stored as e.g. "fileinfo/streamdetails/video/0/codec". const flatSep = "/" @@ -33,7 +33,7 @@ func Reconstruct(flat map[string]string) interface{} { } // Flatten is the inverse of Reconstruct: walks a nested value and produces a -// flat property map whose values are all strings (Redis-ready). +// flat property map whose values are all strings (store-ready). // // Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing // values are stored as "" to preserve the field's presence — this matches @@ -127,7 +127,7 @@ func flattenInto(v interface{}, path string, out map[string]string) { out[path] = "" case map[string]interface{}: if len(t) == 0 && path != "" { - // Empty objects don't have a Redis representation; skip. + // Empty objects don't have a flat-key representation; skip. return } for k, child := range t { diff --git a/internal/snapshot/import.go b/internal/snapshot/import.go index 50f2523..997d14e 100644 --- a/internal/snapshot/import.go +++ b/internal/snapshot/import.go @@ -73,7 +73,7 @@ type ImportResult struct { PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"` } -// Importer applies snapshot ZIPs to a live Redis store. +// Importer applies snapshot ZIPs to the live metadata store. type Importer struct { storage *storage.Client } @@ -83,7 +83,7 @@ func NewImporter(stor *storage.Client) *Importer { } // Import reads a ZIP from r (which must support io.ReaderAt; callers pass -// *bytes.Reader after buffering the upload) and applies it to Redis. +// *bytes.Reader after buffering the upload) and applies it to the store. func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) { if i.storage == nil || !i.storage.IsConnected() { return nil, fmt.Errorf("storage not connected") diff --git a/internal/snapshot/wipe.go b/internal/snapshot/wipe.go index 21d6172..8125c19 100644 --- a/internal/snapshot/wipe.go +++ b/internal/snapshot/wipe.go @@ -16,7 +16,7 @@ type WipeResult struct { MetadataDeleted int64 `json:"metadataDeleted"` } -// Wiper clears metadata from Redis. +// Wiper clears metadata from the store. type Wiper struct { storage *storage.Client } diff --git a/internal/storage/cid_resolution.go b/internal/storage/cid_resolution.go index 07beac9..5559776 100644 --- a/internal/storage/cid_resolution.go +++ b/internal/storage/cid_resolution.go @@ -5,10 +5,9 @@ import ( "fmt" "strconv" "strings" - "time" "github.com/metazla/meta-core/internal/cid" - "github.com/redis/go-redis/v9" + "github.com/metazla/meta-core/internal/kv" ) // This file implements the CID-resolution layer that lives on top of the @@ -18,7 +17,7 @@ import ( // The UUID is an internal implementation detail; the mental model is // "N CIDs → one record of keys→metadata". // - Every known CID for a file is registered as a reverse-index entry: -// cid: → (Redis STRING) +// cid: → // and as a member of the per-root bare-CID key-set: // file:/cids/ = "true" (flat key, sentinel value) // - A CID is self-describing: its algorithm is recoverable from the CIDv1 @@ -27,14 +26,13 @@ import ( // *derived by rank* from the key-set on read (GetMetadataDocument), never // stored. // -// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch -// any write of a cids/ field and register the reverse-index entry, so +// The record write path (writeFieldsTx) catches any write of a cids/ +// field and registers the reverse-index entry in the same transaction, so // writers that just emit the key-set members don't need to know about the -// reverse index at all. The hook helpers are *Locked — they assume c.mu is -// already held by the caller, avoiding nested RLock (which can deadlock -// against a waiting writer). +// reverse index at all. The helpers here take the caller's *kv.Tx: a write +// must never open a second transaction (there is one writer connection). -// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined +// CIDIndexPrefix is the key prefix for the reverse index. Combined // with a bare CID, it forms the full key: "cid:bafk…". const CIDIndexPrefix = "cid:" @@ -45,152 +43,112 @@ const CIDIndexPrefix = "cid:" // reverse-index entries that need cleanup). const CIDsKeyPrefix = "cids/" -// DuplicatesField is the per-root SET key holding additional file paths -// whose content hashes to the same CIDs as this root. The watcher writes -// here when it sees a new path for content that's already indexed; meta-dup -// reads from it. -const DuplicatesField = "duplicates" - -// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry. -// cidStr is a bare multibase CIDv1 ("bafk…"). -func (c *Client) buildCIDIndexKey(cidStr string) string { - return c.buildKey(CIDIndexPrefix + cidStr) -} +// Duplicate paths — other physical locations whose content hashes to the same +// CIDs as a root — live as dup:/ = "" (a key-set). The watcher +// writes one when it sees a new path for content that's already indexed. // Mint creates a new root with filePath/sizeByte/mtimeNano and registers it -// in the file index. Returns the new UUID. The caller is expected to follow +// in the root index. Returns the new UUID. The caller is expected to follow // up with AddAlias for at least one CID — a root without any alias is // reachable only via the index, which works but defeats the point. func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return "", fmt.Errorf("not connected") - } + var uuid string + err := c.update(func(tx *kv.Tx) error { + var err error + uuid, err = mintTx(tx, filePath, size, mtimeNano) + return err + }) + return uuid, err +} +func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) { uuid, err := NewUUID() if err != nil { return "", fmt.Errorf("uuid: %w", err) } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - prefix := c.buildKeyPrefix(uuid) - pipe := c.client.Pipeline() - pipe.Set(ctx, prefix+"filePath", filePath, 0) - pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0) - pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0) - pipe.SAdd(ctx, c.buildIndexKey(), uuid) - c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"}) - if _, err := pipe.Exec(ctx); err != nil { - return "", fmt.Errorf("mint pipeline: %w", err) - } - return uuid, nil + return uuid, writeFieldsTx(tx, uuid, map[string]string{ + "filePath": filePath, + "sizeByte": strconv.FormatInt(size, 10), + "mtimeNano": strconv.FormatInt(mtimeNano, 10), + }) } // AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse // index cid: → uuid and the key-set member cids/ = "true". // Idempotent: registering the same CID twice is a no-op. func (c *Client) AddAlias(uuid, cidStr string) error { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - return c.addAliasLocked(ctx, uuid, cidStr) -} - -// addAliasLocked is the lock-free body of AddAlias. The caller must already -// hold c.mu (read or write). Split out so the alias-maintenance hooks -// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse -// the work without re-entering the lock — nested RLock can deadlock against -// a waiting writer. -func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error { if uuid == "" || cidStr == "" { return fmt.Errorf("uuid and cid are required") } + return c.update(func(tx *kv.Tx) error { + return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"}) + }) +} - keysetField := CIDsKeyPrefix + cidStr - keysetKey := c.buildKeyPrefix(uuid) + keysetField - - // Every branch below writes the key-set member, so index the field name - // once here — including the two guard branches that skip the alias itself. - // Missing it would leave `cids/` invisible to a record read. - defer c.addFieldLocked(ctx, uuid, keysetField) - +// addAliasTx registers the reverse-index entry for a cids/ member the +// caller is writing (writeFieldsTx writes the member itself). +func addAliasTx(tx *kv.Tx, uuid, cidStr string) error { + if uuid == "" || cidStr == "" { + return fmt.Errorf("uuid and cid are required") + } // Self-pointing alias guard. If the caller is trying to register // cid: → (i.e. the root IS the cid value), that's the // dual-root failure mode in disguise — it means somebody wrote // /meta/ without first resolving via ResolveRoot. Refuse the alias - // write; leave any existing alias alone. Still record the key-set - // membership so the field-level write succeeds. + // write; leave any existing alias alone. The key-set membership is still + // recorded by the caller, so the field-level write succeeds. if uuid == cidStr { - return c.client.Set(ctx, keysetKey, "true", 0).Err() + return nil } - // Guard against the dual-root pattern: if cid: already points at a - // DIFFERENT uuid, don't silently overwrite. Leave the existing alias and - // just record the key-set membership so it's still discoverable via the - // root → cids enumeration. - indexKey := c.buildCIDIndexKey(cidStr) - existing, err := c.client.Get(ctx, indexKey).Result() - if err != nil && err != redis.Nil { - return fmt.Errorf("read existing alias: %w", err) - } - if existing != "" && existing != uuid { - return c.client.Set(ctx, keysetKey, "true", 0).Err() - } - - pipe := c.client.Pipeline() - pipe.Set(ctx, indexKey, uuid, 0) - pipe.Set(ctx, keysetKey, "true", 0) - if _, err := pipe.Exec(ctx); err != nil { - return fmt.Errorf("add alias pipeline: %w", err) - } - return nil -} - -// removeAliasLocked is the inverse of addAliasLocked: it drops the reverse-index -// entry cid: → . The caller must already hold c.mu, and must delete -// the cids/ key-set member itself (DeleteProperty does both). -// -// ⚠ Why this exists. A cids/ member write touches THREE keys — the flat -// file:/cids/, the __fields__ index entry, and this reverse alias — -// while the delete path only ever removed the first two. A "deleted" CID -// therefore still resolved: GET /meta/, ResolveRoot and GetByCID all kept -// answering with the record. The asymmetry is not theoretical. meta-share -// published dag-pb roots computed over RAR volumes and par2 files onto six -// records on watch.nsl.sh; removing the member was expressible, removing the -// claim was not, so the false identity survived its own retraction. -// -// Guarded, not blind: the alias is removed only when it currently points at -// THIS uuid. Two records can legitimately race to claim a cid (see the -// dual-root guard in addAliasLocked, which leaves the first writer's alias in -// place); deleting a member from the loser must not unpick the winner's index. -func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) error { - if uuid == "" || cidStr == "" { - return nil - } - indexKey := c.buildCIDIndexKey(cidStr) - existing, err := c.client.Get(ctx, indexKey).Result() - if err == redis.Nil { - return nil - } + // DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the + // key-set membership keeps the CID discoverable via root → cids. + indexKey := CIDIndexPrefix + cidStr + existing, _, err := tx.Get(tx.Context(), indexKey) if err != nil { return fmt.Errorf("read existing alias: %w", err) } - if existing != uuid { - // Somebody else owns this alias. Leave it. + if existing != "" && existing != uuid { return nil } - return c.client.Del(ctx, indexKey).Err() + if existing == uuid { + return nil + } + return tx.Put(indexKey, uuid) +} + +// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index +// entry cid: → . The caller deletes the cids/ key-set member +// itself (DeleteProperty does both). +// +// ⚠ Why this exists. A cids/ member write touches two keys — the flat +// file:/cids/ and this reverse alias — while the delete path once +// only removed the first. A "deleted" CID therefore still resolved: GET +// /meta/, ResolveRoot and GetByCID all kept answering with the record. +// The asymmetry is not theoretical. meta-share published dag-pb roots computed +// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing +// the member was expressible, removing the claim was not, so the false +// identity survived its own retraction. +// +// Guarded, not blind: the alias is removed only when it currently points at +// THIS uuid. Two records can legitimately race to claim a cid (see the +// dual-root guard in addAliasTx, which leaves the first writer's alias in +// place); deleting a member from the loser must not unpick the winner's index. +func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error { + if uuid == "" || cidStr == "" { + return nil + } + indexKey := CIDIndexPrefix + cidStr + existing, ok, err := tx.Get(tx.Context(), indexKey) + if err != nil { + return fmt.Errorf("read existing alias: %w", err) + } + if !ok || existing != uuid { + // Absent, or somebody else owns this alias. Leave it. + return nil + } + _, err = tx.Delete(indexKey) + return err } // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor @@ -210,8 +168,8 @@ func (c *Client) removeAliasLocked(ctx context.Context, uuid, cidStr string) err // - hash matches a reverse-index entry → return UUID // - otherwise return hash unchanged (legacy / explicit-UUID writes) // -// Errors are converted to "no resolution" — if Redis is down or the lookup -// fails we let the caller proceed with the bare hash and surface the Redis +// Errors are converted to "no resolution" — if the store is down or the +// lookup fails we let the caller proceed with the bare hash and surface the // failure on the actual operation. func (c *Client) ResolveRoot(hash string) string { if hash == "" { @@ -228,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string { // when the CID is unknown; the empty string is the "not found" signal rather // than a typed error so callers can branch without errors.As. func (c *Client) GetByCID(cidStr string) (string, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return "", fmt.Errorf("not connected") + s, err := c.kv() + if err != nil { + return "", err } if cidStr == "" { return "", nil } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := opCtx() defer cancel() - - uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result() - if err == redis.Nil { - return "", nil - } + uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr) if err != nil { return "", fmt.Errorf("get cid index: %w", err) } @@ -252,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) { } // AddDuplicatePath records that path is another physical location whose -// content matches uuid. Returns true if path was newly added (not already -// in the set). The watcher calls this when it sees a file whose midhash -// matches an existing root but at a different filePath. +// content matches uuid. Returns true if path was newly added. The watcher +// calls this when it sees a file whose midhash matches an existing root but +// at a different filePath. func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return false, fmt.Errorf("not connected") - } if uuid == "" || path == "" { return false, nil } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result() - if err != nil { - return false, fmt.Errorf("sadd duplicates: %w", err) - } - return added > 0, nil + added := false + err := c.update(func(tx *kv.Tx) error { + var err error + added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "") + return err + }) + return added, err } -// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/ -// prefix stripped). Caller must hold c.mu. -// -// Reads the field index and filters by prefix; only an un-indexed root falls -// back to a SCAN, and then over the narrower cids/ glob. See field_index.go. -func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) { - return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix) +// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix +// stripped). +func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) { + prefix := recordPrefix(uuid) + CIDsKeyPrefix + rows, err := r.Range(ctx, prefix, "", 0) + if err != nil { + return nil, err + } + out := make([]string, 0, len(rows)) + for _, row := range rows { + out = append(out, row.K[len(prefix):]) + } + return out, nil } // DeleteRoot removes everything keyed off uuid: every file:/* property // (including the cids/ key-set), every reverse-index entry for those -// CIDs, and the index membership. Idempotent — a uuid with no remaining keys -// is a successful no-op. +// CIDs, its duplicates, search row and index membership. Idempotent — a uuid +// with no remaining keys is a successful no-op. func (c *Client) DeleteRoot(uuid string) error { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return fmt.Errorf("not connected") - } if uuid == "" { return nil } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - prefix := c.buildKeyPrefix(uuid) - - // 1. Read the cids key-set first — once we delete file:/* we lose - // the list of reverse-index entries to clean up. - cids, err := c.cidsForRootLocked(ctx, uuid) - if err != nil { + return c.update(func(tx *kv.Tx) error { + _, err := deleteRecordTx(tx, uuid) return err - } - - // 2. Delete every reverse-index entry in one shot. - if len(cids) > 0 { - keys := make([]string, 0, len(cids)) - for _, cidStr := range cids { - keys = append(keys, c.buildCIDIndexKey(cidStr)) - } - if err := c.client.Del(ctx, keys...).Err(); err != nil { - return fmt.Errorf("del cid index: %w", err) - } - } - - // 3. Scan and delete every file:/* key (cids/ keys included). - var cursor uint64 - for { - keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result() - if err != nil { - return fmt.Errorf("scan root keys: %w", err) - } - if len(keys) > 0 { - if err := c.client.Del(ctx, keys...).Err(); err != nil { - return fmt.Errorf("del root keys: %w", err) - } - } - cursor = next - if cursor == 0 { - break - } - } - - // 4. Remove from the file index set. - if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil { - return fmt.Errorf("srem index: %w", err) - } - // Marked so the incremental search-index refresh reads it, finds it gone, - // and evicts it — otherwise a deleted root keeps answering searches until - // the next full rebuild. - c.markDirtyLocked(ctx, uuid) - return nil -} - -// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a -// root. If it's a key-set member — field name starts with "cids/" — it -// registers the implied CID in the reverse index. Writers emit cids/ -// members without knowing about the reverse index; this hook keeps it in -// sync transparently. -// -// Caller must hold c.mu. Errors are intentionally swallowed at the call site -// (logged, not returned): an out-of-sync reverse index is recoverable via a -// sweep, while a failed property write is not. -func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error { - cidStr := cidFromKeysetField(field) - if cidStr == "" { - return nil - } - return c.addAliasLocked(ctx, uuid, cidStr) -} - -// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked -// over every entry in a metadata map. Used by SetMetadataFlat / -// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must -// hold c.mu. -func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) { - for k := range metadata { - _ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k]) - } + }) } // cidFromKeysetField returns the bare CID encoded in a cids/ field name, @@ -406,24 +275,23 @@ type MetadataDocument struct { // the duplicates SET. Returns nil if the root has no keys at all (deleted or // never existed). func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return nil, fmt.Errorf("not connected") + s, err := c.kv() + if err != nil { + return nil, err } if uuid == "" { return nil, nil } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := opCtx() defer cancel() - flat, err := c.getMetadataFlatInternal(ctx, uuid) + flat, err := readRecord(ctx, s, uuid) if err != nil { return nil, err } - prefix := c.buildKeyPrefix(uuid) + if flat == nil { + flat = make(map[string]string) + } // Hoist the cids/ key-set members into a typed array and derive the // canonical CID by rank. The members remain in Flat too — that's how @@ -443,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { } } - dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result() - if err != nil && err != redis.Nil { - return nil, fmt.Errorf("smembers duplicates: %w", err) + dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0) + if err != nil { + return nil, fmt.Errorf("read duplicates: %w", err) + } + dups := make([]string, 0, len(dupRows)) + for _, row := range dupRows { + dups = append(dups, row.K[len(dupPrefix(uuid)):]) } if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 { @@ -474,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas if midhash == "" { return "", false, fmt.Errorf("midhash is required") } - uuid, err := c.GetByCID(midhash) - if err != nil { - return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err) - } - - if uuid == "" { - uuid, err = c.Mint(absPath, size, mtimeNano) + var ( + uuid string + minted bool + ) + // One transaction: the lookup and the mint cannot interleave with another + // registration of the same content, so two concurrent scans of one file + // can no longer mint two roots for it. + err := c.update(func(tx *kv.Tx) error { + ctx := tx.Context() + existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash) if err != nil { - return "", false, fmt.Errorf("Mint(%s): %w", absPath, err) + return fmt.Errorf("GetByCID(%s): %w", midhash, err) } - if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil { - return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err) + uuid = existing + if uuid == "" { + if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil { + return fmt.Errorf("Mint(%s): %w", absPath, err) + } + minted = true + if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil { + return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err) + } + return nil } - return uuid, true, nil - } - - existing, _ := c.GetProperty(uuid, "filePath") - if existing != absPath { - if _, err := c.AddDuplicatePath(uuid, absPath); err != nil { - return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err) + current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath") + if err != nil { + return err } - return uuid, false, nil + if current != absPath { + if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil { + return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err) + } + return nil + } + if err := writeFieldsTx(tx, uuid, map[string]string{ + "sizeByte": strconv.FormatInt(size, 10), + "mtimeNano": strconv.FormatInt(mtimeNano, 10), + }); err != nil { + return fmt.Errorf("refresh tuple for %s: %w", uuid, err) + } + return nil + }) + if err != nil { + return "", false, err } - if _, err := c.MergeMetadataFlat(uuid, map[string]string{ - "sizeByte": strconv.FormatInt(size, 10), - "mtimeNano": strconv.FormatInt(mtimeNano, 10), - }); err != nil { - return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err) - } - return uuid, false, nil + return uuid, minted, nil } diff --git a/internal/storage/cid_retraction_test.go b/internal/storage/cid_retraction_test.go index c139842..c3f1dd9 100644 --- a/internal/storage/cid_retraction_test.go +++ b/internal/storage/cid_retraction_test.go @@ -6,13 +6,13 @@ import ( ) // Tests for retracting a `cids/` key-set member (cid_resolution.go's -// removeAliasLocked, reached through DeleteProperty). +// removeAliasTx, reached through DeleteProperty). // -// Why this is worth its own file. Adding a member writes THREE keys — the flat -// `file:/cids/`, the `__fields__` index entry, and the reverse alias -// `cid:` → ``. Until removeAliasLocked existed, deleting removed only -// the first two, so a retracted CID kept resolving to the record: the member was -// gone from the document and the claim was not. +// Why this is worth its own file. Adding a member writes two keys — the flat +// `file:/cids/` and the reverse alias `cid:` → ``. Until +// removeAliasLocked (now removeAliasTx) existed, deleting removed only the +// first, so a retracted CID kept resolving to the record: the member was gone +// from the document and the claim was not. // // That asymmetry had a live consequence. meta-share minted dag-pb roots and // midhash256s over whatever `archive::pick_media` elected, which on an @@ -26,19 +26,20 @@ import ( // aliasTarget returns the uuid that cid: currently resolves to, or "". func aliasTarget(t *testing.T, c *Client, cidStr string) string { t.Helper() - got, err := c.client.Get(context.Background(), c.buildCIDIndexKey(cidStr)).Result() + got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr) if err != nil { - return "" + t.Fatalf("get alias: %v", err) } return got } func hasField(t *testing.T, c *Client, hashID, field string) bool { t.Helper() - ok, err := c.client.SIsMember(context.Background(), c.buildFieldsKey(hashID), field).Result() + fields, err := c.GetMetadataFlat(hashID) if err != nil { - t.Fatalf("sismember: %v", err) + t.Fatalf("read record: %v", err) } + _, ok := fields[field] return ok } @@ -50,7 +51,7 @@ const ( ) func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) { - forEachPrefix(t, func(t *testing.T, c *Client) { + withClient(t, func(t *testing.T, c *Client) { const uuid = "01KSBHMTESTRECORD" if err := c.SetMetadataFlat(uuid, map[string]string{ @@ -77,13 +78,13 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) { } // 1. the flat key - if v, err := c.client.Get(context.Background(), - c.buildKeyPrefix(uuid)+"cids/"+poisonedRoot).Result(); err == nil { + if v, ok, _ := c.Store().Get(context.Background(), + recordPrefix(uuid)+"cids/"+poisonedRoot); ok { t.Fatalf("flat key should be gone, still %q", v) } - // 2. the field index + // 2. the record read if hasField(t, c, uuid, "cids/"+poisonedRoot) { - t.Fatal("field index still lists the retracted member") + t.Fatal("record read still lists the retracted member") } // 3. the reverse alias — the one that used to survive if got := aliasTarget(t, c, poisonedRoot); got != "" { @@ -110,7 +111,7 @@ func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) { // writer's alias in place. Retracting the loser's member must not unpick the // winner's index — which is why removeAliasLocked compares before deleting. func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) { - forEachPrefix(t, func(t *testing.T, c *Client) { + withClient(t, func(t *testing.T, c *Client) { const winner = "01KSBHMWINNER" const loser = "01KSBHMLOSER" const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m" @@ -140,7 +141,7 @@ func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) { // A non-cids field has no alias to clean up, and the delete must not invent one. func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) { - forEachPrefix(t, func(t *testing.T, c *Client) { + withClient(t, func(t *testing.T, c *Client) { const uuid = "01KSBHMORDINARY" if err := c.SetMetadataFlat(uuid, map[string]string{ "cids/" + goodMidhash: "true", diff --git a/internal/storage/clear_test.go b/internal/storage/clear_test.go index 71260a8..f00d5a0 100644 --- a/internal/storage/clear_test.go +++ b/internal/storage/clear_test.go @@ -1,111 +1,122 @@ package storage import ( + "context" + "path/filepath" + "strings" "testing" - "github.com/alicebob/miniredis/v2" + "github.com/metazla/meta-core/internal/kv" ) -// newTestClient spins an in-memory Redis and returns a connected Client with -// the given namespace prefix. The server is torn down with the test. -func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) { +// newTestClient opens a fresh database in the test's temp dir. +func newTestClient(t *testing.T) *Client { t.Helper() - - s := miniredis.RunT(t) - c := NewClient(prefix) - if err := c.Connect("redis://" + s.Addr()); err != nil { - t.Fatalf("connect to miniredis: %v", err) + c := NewClient() + if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil { + t.Fatalf("connect: %v", err) } t.Cleanup(func() { _ = c.Close() }) - return c, s + return c +} + +// allKeys lists every row (test helper; full scan). +func allKeys(t *testing.T, c *Client) []string { + t.Helper() + var keys []string + if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error { + for _, r := range rows { + keys = append(keys, r.K) + } + return nil + }); err != nil { + t.Fatalf("scan: %v", err) + } + return keys } // TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the -// leak: ClearAllMetadata used to delete every file:/* key and the -// file:__index__ set, but left the cid: → root reverse-index entries -// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so -// a surviving entry silently redirects a later write into a deleted root. +// leak: ClearAllMetadata used to delete every record key and the root index, +// but left the cid: → root reverse-index entries behind. Those stale +// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry +// silently redirects a later write into a deleted root. func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) { - for _, prefix := range []string{"", "mm:"} { - name := "no-prefix" - if prefix != "" { - name = "prefixed" + c := newTestClient(t) + + // Two records, each with several CID aliases (a midhash plus siblings) — + // exactly what the watcher produces. + aliases := map[string][]string{} + for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} { + uuid, err := c.Mint(f, int64(100+i), int64(1000+i)) + if err != nil { + t.Fatalf("mint: %v", err) } - t.Run(name, func(t *testing.T) { - c, s := newTestClient(t, prefix) + cids := []string{"bafkmid" + uuid, "bafkfull" + uuid} + for _, cidStr := range cids { + if err := c.AddAlias(uuid, cidStr); err != nil { + t.Fatalf("add alias %s: %v", cidStr, err) + } + } + if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil { + t.Fatalf("add duplicate: %v", err) + } + aliases[uuid] = cids + } - // Two records, each with several CID aliases (a midhash plus - // siblings) — exactly what the watcher produces. - aliases := map[string][]string{} - for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} { - uuid, err := c.Mint(f, int64(100+i), int64(1000+i)) - if err != nil { - t.Fatalf("mint: %v", err) - } - cids := []string{ - "bafkmid" + uuid, - "bafkfull" + uuid, - } - for _, cidStr := range cids { - if err := c.AddAlias(uuid, cidStr); err != nil { - t.Fatalf("add alias %s: %v", cidStr, err) - } - } - aliases[uuid] = cids - } + // An entry already orphaned by a previous (buggy) clear: a reverse-index + // key whose root no longer exists. The sweep is expected to collect it too. + if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil { + t.Fatalf("seed orphan: %v", err) + } - // An entry already orphaned by a previous (buggy) clear: a - // reverse-index key whose root no longer exists. The sweep is - // expected to collect it too. - if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil { - t.Fatalf("seed orphan: %v", err) + for uuid, cids := range aliases { + for _, cidStr := range cids { + if got := c.ResolveRoot(cidStr); got != uuid { + t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid) } + } + } - // Sanity: the aliases resolve before the clear. - for uuid, cids := range aliases { - for _, cidStr := range cids { - if got := c.ResolveRoot(cidStr); got != uuid { - t.Fatalf("pre-clear ResolveRoot(%s) = %q, want %q", cidStr, got, uuid) - } - } - } + deleted, err := c.ClearAllMetadata() + if err != nil { + t.Fatalf("ClearAllMetadata: %v", err) + } + if deleted != 2 { + t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted) + } - deleted, err := c.ClearAllMetadata() - if err != nil { - t.Fatalf("ClearAllMetadata: %v", err) - } - // deletedCount counts files, not keys. - if deleted != 2 { - t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted) + // Nothing record-shaped may survive: not the records, not their aliases, + // duplicates or search rows, not the pre-existing orphan. (The event + // streams and the schema sentinel are not metadata and stay.) + for _, k := range allKeys(t, c) { + for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} { + if strings.HasPrefix(k, ns) { + t.Errorf("key survived the clear: %q", k) } + } + } + var ftsRows int + if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 { + t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err) + } - // No reverse-index key may survive — not for the cleared records, - // not the pre-existing orphan. - if leaked := s.Keys(); len(leaked) != 0 { - t.Errorf("keys survived the clear: %v", leaked) + for _, cids := range aliases { + for _, cidStr := range cids { + if got := c.ResolveRoot(cidStr); got != cidStr { + t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got) } - - // And the aliases must no longer resolve: ResolveRoot returns the - // hash unchanged when there is no index entry. - for _, cids := range aliases { - for _, cidStr := range cids { - if got := c.ResolveRoot(cidStr); got != cidStr { - t.Errorf("post-clear ResolveRoot(%s) = %q, want the hash unchanged (stale alias)", cidStr, got) - } - } - } - if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" { - t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err) - } - }) + } + } + if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" { + t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err) } } // TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius: -// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:", -// so a clear on a prefixed client cannot reach another namespace's index. +// keys outside the metadata namespaces (magnet cache, coverage markers, a key +// that merely CONTAINS "cid:") must survive a metadata wipe. func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) { - c, s := newTestClient(t, "mm:") + c := newTestClient(t) uuid, err := c.Mint("watch/a.mkv", 100, 1000) if err != nil { @@ -114,25 +125,28 @@ func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) { if err := c.AddAlias(uuid, "bafkmine"); err != nil { t.Fatalf("add alias: %v", err) } - - // Another namespace's reverse-index entry, plus an unrelated key. - if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil { - t.Fatalf("seed foreign cid: %v", err) - } - if err := s.Set("unrelated", "keep-me"); err != nil { - t.Fatalf("seed unrelated: %v", err) + for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} { + if err := c.SetRawValue(k, v); err != nil { + t.Fatalf("seed %s: %v", k, err) + } } if _, err := c.ClearAllMetadata(); err != nil { t.Fatalf("ClearAllMetadata: %v", err) } - for _, k := range []string{"other:cid:bafktheirs", "unrelated"} { - if !s.Exists(k) { - t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k) + for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} { + if _, _, ok, _ := c.GetKeyInfo(k); !ok { + t.Errorf("clear deleted foreign key %q", k) } } - if s.Exists(c.buildCIDIndexKey("bafkmine")) { + if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok { t.Errorf("own reverse-index entry survived the clear") } } + +// withClient runs fn against a fresh client. +func withClient(t *testing.T, fn func(t *testing.T, c *Client)) { + t.Helper() + fn(t, newTestClient(t)) +} diff --git a/internal/storage/client.go b/internal/storage/client.go index c1703c9..09fbf02 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -2,351 +2,362 @@ package storage import ( "context" + "database/sql" + "errors" "fmt" "log" - "os" "sort" "strconv" "strings" "sync" "time" - "github.com/redis/go-redis/v9" + "github.com/metazla/meta-core/internal/kv" ) -// Client wraps Redis operations for metadata storage +// Key layout (everything is one string→string table, see internal/kv): +// +// file:/ a record property — the ONLY thing under `file:` +// cid: reverse index → +// idx:roots/ the set of every root (was the Redis SET file:__index__) +// idx:fts/ rowid of the record's row in the FTS table +// dup:/ a duplicate path of the record (was the SET file:/duplicates) +// udl:… User Data Layer, see udl.go +// ev:/…, evhwm:… event streams, see internal/kv/stream.go +// +// Keeping bookkeeping OUT of `file:` is load-bearing: the `file:` range is +// exactly the records, so a record is the key range [file:/, file:0), +// the field expression index covers only real fields, and every change under +// `file:` is a real field change worth publishing on meta:events. +const ( + recordNS = "file:" + rootsNS = "idx:roots/" + ftsNS = "idx:fts/" + dupNS = "dup:" + streamNS = "ev:" + hwmNS = "evhwm:" + opTimeout = 30 * time.Second + bulkBatch = 10_000 + readerPage = 5_000 +) + +// MetaEventsStream receives a set/del entry for every record-key change. It is +// written by the storage engine in the same transaction as the change (the +// replacement for Redis keyspace notifications), and served as SSE by +// /api/events/meta. +const MetaEventsStream = "meta:events" + +// MetaEventsMaxLen is the approximate cap on meta:events. Bounded retention is +// a contract requirement for the SSE wire — see docs/api-mediated-access.md +// "Backing-store retention and lifecycle". +const MetaEventsMaxLen = 100_000 + +// Client is meta-core's metadata store. type Client struct { - client *redis.Client - prefix string - connected bool - mu sync.RWMutex + store *kv.Store + mu sync.RWMutex + + keysMu sync.Mutex + keysCount int64 + keysAt time.Time } -// NewClient creates a new storage client -func NewClient(prefix string) *Client { - return &Client{ - prefix: prefix, - } +// NewClient creates an unconnected client. +func NewClient() *Client { + return &Client{} } -// Connect connects to Redis at the given URL -func (c *Client) Connect(url string) error { +// Connect opens (creating if needed) the database file at path. +func (c *Client) Connect(path string) error { c.mu.Lock() defer c.mu.Unlock() - // Parse Redis URL (redis://host:port) - addr := strings.TrimPrefix(url, "redis://") - - c.client = redis.NewClient(&redis.Options{ - Addr: addr, - DialTimeout: 5 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - PoolSize: 10, + store, err := kv.Open(path, kv.Options{ + EventStream: MetaEventsStream, + EventMaxLen: MetaEventsMaxLen, + EventKey: func(k string) bool { return strings.HasPrefix(k, recordNS) }, + WriterCacheMB: envInt("META_CORE_DB_CACHE_MB", 64), + ReaderCacheMB: envInt("META_CORE_DB_READER_CACHE_MB", 16), + Readers: envInt("META_CORE_DB_READERS", 8), + Schema: searchSchema, }) - - // Test connection - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - if err := c.client.Ping(ctx).Err(); err != nil { - return fmt.Errorf("failed to connect to Redis: %w", err) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) } - - c.connected = true - log.Printf("[Storage] Connected to Redis at %s", addr) + c.store = store + log.Printf("[Storage] Opened %s", path) return nil } -// Close closes the Redis connection +// Close closes the database. func (c *Client) Close() error { c.mu.Lock() defer c.mu.Unlock() - - if c.client != nil { - c.connected = false - return c.client.Close() + if c.store == nil { + return nil } - return nil + err := c.store.Close() + c.store = nil + return err } -// IsConnected returns true if connected to Redis +// IsConnected reports whether the database is open. func (c *Client) IsConnected() bool { c.mu.RLock() defer c.mu.RUnlock() - return c.connected + return c.store != nil } -// Health checks if Redis is responding +// Health checks that the database answers. func (c *Client) Health() bool { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil || !c.connected { + s, err := c.kv() + if err != nil { return false } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - - return c.client.Ping(ctx).Err() == nil + return s.Ping(ctx) == nil } -// buildKey constructs the full key with prefix -func (c *Client) buildKey(key string) string { - if c.prefix != "" { - return c.prefix + key - } - return key -} - -// buildKeyPrefix constructs the prefix for a file's flat keys -func (c *Client) buildKeyPrefix(hashID string) string { - return c.buildKey(fmt.Sprintf("file:%s/", hashID)) -} - -// buildIndexKey constructs the key for the file index set -func (c *Client) buildIndexKey() string { - return c.buildKey("file:__index__") -} - -// GetMetadataFlat retrieves all metadata for a file as a flat map. -// -// Reads the record's field-name index and MGETs those keys — two round-trips, -// independent of keyspace size. Records written before the index existed fall -// back to the old SCAN and are indexed on the way out. See field_index.go for -// why the index exists and why it is safe to derive. -func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) { +// Store exposes the underlying engine (event streams for SSE and the schema +// indexer). +func (c *Client) Store() *kv.Store { c.mu.RLock() defer c.mu.RUnlock() + return c.store +} - if c.client == nil { +func (c *Client) kv() (*kv.Store, error) { + c.mu.RLock() + defer c.mu.RUnlock() + if c.store == nil { return nil, fmt.Errorf("not connected") } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - return c.readMetadataLocked(ctx, hashID) + return c.store, nil } -// SetMetadataFlat stores metadata for a file using flat keys -// Uses pipeline SET for each key: file:{hashId}/{property} -func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return fmt.Errorf("not connected") +// update runs fn in one write transaction with the standard timeout. +func (c *Client) update(fn func(tx *kv.Tx) error) error { + s, err := c.kv() + if err != nil { + return err } - - if len(metadata) == 0 { - return nil - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), opTimeout) defer cancel() + return s.Update(ctx, fn) +} - prefix := c.buildKeyPrefix(hashID) - pipe := c.client.Pipeline() +func opCtx() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), opTimeout) +} - fields := make([]string, 0, len(metadata)) +func recordPrefix(hashID string) string { return recordNS + hashID + "/" } +func rootKey(hashID string) string { return rootsNS + hashID } +func dupPrefix(hashID string) string { return dupNS + hashID + "/" } + +// isReservedFieldName reports whether field names bookkeeping rather than +// metadata. `__fields__` was the Redis-era per-record field index; it no +// longer exists, but a payload carrying it (an old snapshot) is still refused +// so it can never become a phantom field. +func isReservedFieldName(field string) bool { + return field == "__fields__" +} + +// splitRecordKey decomposes a key into (hashID, field) when it is a record +// property key — `file:/`, split at the FIRST slash so CID-rooted +// and UUID-rooted records parse alike. +func splitRecordKey(key string) (hashID, field string, ok bool) { + rest, found := strings.CutPrefix(key, recordNS) + if !found { + return "", "", false + } + hashID, field, found = strings.Cut(rest, "/") + if !found || hashID == "" || field == "" || isReservedFieldName(field) { + return "", "", false + } + return hashID, field, true +} + +// isInternalKey reports keys that are derived bookkeeping, which the raw KV +// editor must never write (it would desynchronise the derived state). +func isInternalKey(key string) bool { + return strings.HasPrefix(key, "idx:") || strings.HasPrefix(key, dupNS) || + strings.HasPrefix(key, streamNS) || strings.HasPrefix(key, hwmNS) +} + +// readRecord returns a record's flat fields (nil when it has none). +func readRecord(ctx context.Context, r kv.Reader, hashID string) (map[string]string, error) { + prefix := recordPrefix(hashID) + rows, err := r.Range(ctx, prefix, "", 0) + if err != nil { + return nil, fmt.Errorf("read %s: %w", hashID, err) + } + if len(rows) == 0 { + return nil, nil + } + out := make(map[string]string, len(rows)) + for _, row := range rows { + out[row.K[len(prefix):]] = row.V + } + return out, nil +} + +// writeFieldsTx is the one write path for record fields: the keys, the root +// index, the CID reverse aliases and the search row, all in the caller's +// transaction. +func writeFieldsTx(tx *kv.Tx, hashID string, metadata map[string]string) error { + if hashID == "" { + return fmt.Errorf("empty hash id") + } + prefix := recordPrefix(hashID) + touchesSearch := false for field, value := range metadata { - // A payload carrying the reserved name (a hand-made snapshot, a buggy - // client) would SET a string over the index SET and break every - // subsequent read of this record. Drop it — it is derived state, so - // there is never a reason to accept it from outside. if isReservedFieldName(field) { continue } - pipe.Set(ctx, prefix+field, value, 0) - fields = append(fields, field) + if err := tx.Put(prefix+field, value); err != nil { + return err + } + touchesSearch = touchesSearch || affectsHaystack(field) } - // Same pipeline as the SETs, so indexing the field names costs no extra - // round-trip. See field_index.go. - c.addFieldsToPipe(ctx, pipe, hashID, fields) - - if _, err := pipe.Exec(ctx); err != nil { - return fmt.Errorf("pipeline exec failed: %w", err) + if err := tx.Put(rootKey(hashID), ""); err != nil { + return err } - - // Add to index set - if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { - return fmt.Errorf("sadd index failed: %w", err) + for field := range metadata { + if cidStr := cidFromKeysetField(field); cidStr != "" { + if err := addAliasTx(tx, hashID, cidStr); err != nil { + return err + } + } } - c.markDirtyLocked(ctx, hashID) - - // Auto-register reverse-index aliases for any cids/ key-set members in - // the payload. That is the ONLY shape recognised — the legacy flat cid_* / - // midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField). - c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata) - - return nil + if touchesSearch { + return refreshSearchTx(tx, hashID) + } + return ensureSearchRowTx(tx, hashID) } -// GetAllHashIDs returns all unique file hash IDs stored -// Uses Redis Set: SMEMBERS file:__index__ -func (c *Client) GetAllHashIDs() ([]string, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return nil, fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - indexKey := c.buildIndexKey() - result, err := c.client.SMembers(ctx, indexKey).Result() +// GetMetadataFlat retrieves all metadata for a file as a flat map. +func (c *Client) GetMetadataFlat(hashID string) (map[string]string, error) { + s, err := c.kv() if err != nil { - return nil, fmt.Errorf("smembers failed: %w", err) + return nil, err } - - return result, nil + ctx, cancel := opCtx() + defer cancel() + return readRecord(ctx, s, hashID) } -// GetProperty retrieves a single property value -// Uses flat key: GET file:{hashId}/{property} +// SetMetadataFlat stores metadata for a file using flat keys (merge: fields +// absent from metadata are kept — the Redis implementation never cleared +// either). +func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) error { + if len(metadata) == 0 { + return nil + } + return c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }) +} + +// MergeMetadataFlat merges new metadata into existing (PATCH semantics): new +// keys are added, existing keys are updated, missing keys are NOT deleted. +func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) { + if len(metadata) == 0 { + return 0, nil + } + if err := c.update(func(tx *kv.Tx) error { return writeFieldsTx(tx, hashID, metadata) }); err != nil { + return 0, err + } + return len(metadata), nil +} + +// GetAllHashIDs returns every root id, sorted. +func (c *Client) GetAllHashIDs() ([]string, error) { + s, err := c.kv() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout) + defer cancel() + var out []string + err = s.ScanPrefix(ctx, rootsNS, readerPage, func(rows []kv.KV) error { + for _, row := range rows { + out = append(out, row.K[len(rootsNS):]) + } + return nil + }) + return out, err +} + +// GetProperty retrieves a single property value ("" when absent). func (c *Client) GetProperty(hashID, property string) (string, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return "", fmt.Errorf("not connected") + s, err := c.kv() + if err != nil { + return "", err } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := opCtx() defer cancel() - - key := c.buildKeyPrefix(hashID) + property - result, err := c.client.Get(ctx, key).Result() - if err == redis.Nil { - return "", nil - } - return result, err + v, _, err := s.Get(ctx, recordPrefix(hashID)+property) + return v, err } -// SetProperty sets a single property value -// Uses flat key: SET file:{hashId}/{property} {value} +// SetProperty sets a single property value. func (c *Client) SetProperty(hashID, property, value string) error { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // See SetMetadataFlat: the reserved name addresses the derived index SET, - // not a metadata field. if isReservedFieldName(property) { return fmt.Errorf("field name %q is reserved", property) } - - key := c.buildKeyPrefix(hashID) + property - - // Use SET to set the key - if err := c.client.Set(ctx, key, value, 0).Err(); err != nil { - return fmt.Errorf("set failed: %w", err) - } - - c.addFieldLocked(ctx, hashID, property) - - // Add to index set - if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { - return fmt.Errorf("sadd index failed: %w", err) - } - c.markDirtyLocked(ctx, hashID) - - // Auto-register reverse-index alias if this is a CID field. - _ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value) - - return nil + return c.update(func(tx *kv.Tx) error { + return writeFieldsTx(tx, hashID, map[string]string{property: value}) + }) } -// DeleteMetadata deletes all metadata for a file -// Uses SCAN + DEL for flat keys: file:{hashId}/* -// Also unmaps every reverse-index entry registered for this root, so a -// subsequent GET /api/meta/ correctly returns 404 instead of -// pointing at a UUID with no keys. +// deleteRecordTx removes everything keyed off hashID: its properties, the +// reverse-index entries its cids/ members own, its duplicates, its root-index +// entry and its search row. Returns the number of property keys removed. +func deleteRecordTx(tx *kv.Tx, hashID string) (int64, error) { + ctx := tx.Context() + cids, err := cidsForRoot(ctx, tx, hashID) + if err != nil { + return 0, err + } + for _, cidStr := range cids { + if _, err := tx.Delete(CIDIndexPrefix + cidStr); err != nil { + return 0, err + } + } + n, err := tx.DeletePrefix(recordPrefix(hashID), 0) + if err != nil { + return 0, err + } + if _, err := tx.DeletePrefix(dupPrefix(hashID), 0); err != nil { + return n, err + } + if _, err := tx.Delete(rootKey(hashID)); err != nil { + return n, err + } + return n, deleteSearchRowTx(tx, hashID) +} + +// DeleteMetadata deletes all metadata for a file, and unmaps every +// reverse-index entry registered for this root, so a subsequent +// GET /api/meta/ correctly returns 404 instead of pointing at a root +// with no keys. func (c *Client) DeleteMetadata(hashID string) (int64, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return 0, fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - prefix := c.buildKeyPrefix(hashID) - - // Read the cids key-set first — once file:/* is gone we've lost the - // list of reverse-index keys that need cleanup. - cids, err := c.cidsForRootLocked(ctx, hashID) - if err != nil { - return 0, err - } - if len(cids) > 0 { - idxKeys := make([]string, 0, len(cids)) - for _, cidStr := range cids { - idxKeys = append(idxKeys, c.buildCIDIndexKey(cidStr)) - } - if err := c.client.Del(ctx, idxKeys...).Err(); err != nil { - return 0, fmt.Errorf("del cid index: %w", err) - } - } - - // Scan and delete all keys with this prefix - var deletedCount int64 - var cursor uint64 - for { - keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result() - if err != nil { - return deletedCount, fmt.Errorf("scan failed: %w", err) - } - - if len(keys) > 0 { - deleted, err := c.client.Del(ctx, keys...).Result() - if err != nil { - return deletedCount, fmt.Errorf("del failed: %w", err) - } - deletedCount += deleted - } - - cursor = nextCursor - if cursor == 0 { - break - } - } - - // Remove from index set - if err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Err(); err != nil { - return deletedCount, fmt.Errorf("srem index failed: %w", err) - } - // Marked, not unmarked: the incremental refresh reads the record, finds it - // gone, and evicts it from the search index. Skipping this would leave a - // deleted record answering searches until the next full rebuild. - c.markDirtyLocked(ctx, hashID) - - return deletedCount, nil + var n int64 + err := c.update(func(tx *kv.Tx) error { + var err error + n, err = deleteRecordTx(tx, hashID) + return err + }) + return n, err } -// CountFiles returns the number of unique files stored +// CountFiles returns the number of indexed roots. func (c *Client) CountFiles() (int, error) { - hashIDs, err := c.GetAllHashIDs() - if err != nil { - return 0, err - } - return len(hashIDs), nil + n, err := c.CountRoots() + return int(n), err } // LookupPathByCID resolves a bare CID to its file path on disk via the -// reverse index. O(1) lookup followed by a single GET of file:/filePath. +// reverse index: one lookup followed by a single read of file:/filePath. // // Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS // CID, a btih, etc. (they all alias to the same root). The bare CID is what @@ -381,239 +392,92 @@ func normalizeFilesRelativePath(p string) string { return strings.TrimPrefix(p, "/files") } -// getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock. -// Callers must already hold c.mu. -func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) { - result, err := c.readMetadataLocked(ctx, hashID) - if err != nil { - return nil, err - } - // Unlike GetMetadataFlat, this variant hands back an empty map rather than - // nil for a record with no fields — GetMetadataDocument writes into it. - if result == nil { - result = make(map[string]string) - } - return result, nil -} - -// getAllHashIDsInternal is an internal version that doesn't acquire locks -// (caller must hold the lock) -func (c *Client) getAllHashIDsInternal(ctx context.Context) ([]string, error) { - indexKey := c.buildIndexKey() - result, err := c.client.SMembers(ctx, indexKey).Result() - if err != nil { - return nil, fmt.Errorf("smembers failed: %w", err) - } - return result, nil -} - -// MergeMetadataFlat merges new metadata into existing (PATCH semantics) -// New keys are added, existing keys are updated, missing keys are NOT deleted -// Uses pipeline SET for flat keys: file:{hashId}/{property} -func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (int, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return 0, fmt.Errorf("not connected") - } - - if len(metadata) == 0 { - return 0, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - prefix := c.buildKeyPrefix(hashID) - pipe := c.client.Pipeline() - - fields := make([]string, 0, len(metadata)) - for field, value := range metadata { - // See SetMetadataFlat: never let an inbound payload overwrite the - // derived index SET with a string. - if isReservedFieldName(field) { - continue - } - pipe.Set(ctx, prefix+field, value, 0) - fields = append(fields, field) - } - c.addFieldsToPipe(ctx, pipe, hashID, fields) - - if _, err := pipe.Exec(ctx); err != nil { - return 0, fmt.Errorf("pipeline exec failed: %w", err) - } - - // Add to index set - if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { - return len(metadata), fmt.Errorf("sadd index failed: %w", err) - } - c.markDirtyLocked(ctx, hashID) - - // Auto-register reverse-index aliases for any cids/ key-set members in - // the payload. That is the ONLY shape recognised — the legacy flat cid_* / - // midhash256 named fields are not (see cid_resolution.go: cidFromKeysetField). - c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata) - - return len(metadata), nil -} - -// DeleteProperty deletes a single property -// Uses flat key: DEL file:{hashId}/{property} +// DeleteProperty deletes a single property. // -// For a cids/ member this is the full inverse of the write: the flat key, -// the __fields__ entry AND the cid: reverse alias all go. Anything less -// leaves the CID still resolving to this record — see removeAliasLocked. +// For a cids/ member this is the full inverse of the write: the flat key +// AND the cid: reverse alias both go. Anything less leaves the CID still +// resolving to this record — see removeAliasTx. func (c *Client) DeleteProperty(hashID, property string) error { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - key := c.buildKeyPrefix(hashID) + property - if err := c.client.Del(ctx, key).Err(); err != nil { - return err - } - // Before the field index forgets the name, use it: a cids/ member owns a - // reverse-index entry that nothing else will ever clean up. - if cidStr := cidFromKeysetField(property); cidStr != "" { - if err := c.removeAliasLocked(ctx, hashID, cidStr); err != nil { - return fmt.Errorf("remove alias for %s: %w", cidStr, err) + return c.update(func(tx *kv.Tx) error { + if _, err := tx.Delete(recordPrefix(hashID) + property); err != nil { + return err } - } - c.removeFieldLocked(ctx, hashID, property) - c.markDirtyLocked(ctx, hashID) - return nil + if cidStr := cidFromKeysetField(property); cidStr != "" { + if err := removeAliasTx(tx, hashID, cidStr); err != nil { + return fmt.Errorf("remove alias for %s: %w", cidStr, err) + } + } + if affectsHaystack(property) { + return refreshSearchTx(tx, hashID) + } + return nil + }) } -// AddToSet adds a value to a set-type field (stored as pipe-delimited string in flat key) -// Returns true if value was added, false if it already existed +// AddToSet adds a value to a set-type field (stored as a pipe-delimited string +// in one flat key). Returns true if value was added, false if it already +// existed. func (c *Client) AddToSet(hashID, property, value string) (bool, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.client == nil { - return false, fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - key := c.buildKeyPrefix(hashID) + property - - // Get current value - current, err := c.client.Get(ctx, key).Result() - if err != nil && err != redis.Nil { - return false, err - } - - // Parse existing values (pipe-delimited) - var values []string - if current != "" { - values = strings.Split(current, "|") - } - - // Check if value already exists - for _, v := range values { - if v == value { - return false, nil // Already exists + added := false + err := c.update(func(tx *kv.Tx) error { + current, _, err := tx.Get(tx.Context(), recordPrefix(hashID)+property) + if err != nil { + return err } - } - - // Add new value - values = append(values, value) - newValue := strings.Join(values, "|") - - // Save back using SET - if err := c.client.Set(ctx, key, newValue, 0).Err(); err != nil { - return false, err - } - - // The SET above may have created the key, so the field name is new. - c.addFieldLocked(ctx, hashID, property) - - // Add to index set - if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { - return true, fmt.Errorf("sadd index failed: %w", err) - } - c.markDirtyLocked(ctx, hashID) - - return true, nil + var values []string + if current != "" { + values = strings.Split(current, "|") + } + for _, v := range values { + if v == value { + return nil + } + } + added = true + return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(append(values, value), "|")}) + }) + return added, err } -// RemoveFromSet removes a value from a set-type field (stored in flat key) -// Returns true if value was removed, false if it didn't exist +// RemoveFromSet removes a value from a set-type field (stored in one flat key). +// Returns true if value was removed, false if it didn't exist. func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) { - c.mu.Lock() - defer c.mu.Unlock() - - if c.client == nil { - return false, fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - key := c.buildKeyPrefix(hashID) + property - - // Get current value - current, err := c.client.Get(ctx, key).Result() - if err == redis.Nil { - return false, nil // Key doesn't exist - } - if err != nil { - return false, err - } - - // Parse existing values - values := strings.Split(current, "|") - - // Find and remove value - found := false - newValues := make([]string, 0, len(values)) - for _, v := range values { - if v == value { - found = true - } else { - newValues = append(newValues, v) + removed := false + err := c.update(func(tx *kv.Tx) error { + key := recordPrefix(hashID) + property + current, ok, err := tx.Get(tx.Context(), key) + if err != nil || !ok { + return err } - } - - if !found { - return false, nil - } - - // Save back (or delete key if empty) - if len(newValues) == 0 { - if err := c.client.Del(ctx, key).Err(); err != nil { - return true, err + values := strings.Split(current, "|") + kept := make([]string, 0, len(values)) + for _, v := range values { + if v == value { + removed = true + } else { + kept = append(kept, v) + } } - // The key is gone, so the field name must leave the index too. - c.removeFieldLocked(ctx, hashID, property) - c.markDirtyLocked(ctx, hashID) - return true, nil - } - - newValue := strings.Join(newValues, "|") - err = c.client.Set(ctx, key, newValue, 0).Err() - // The field survives with fewer members, so no field-index change — but the - // VALUE changed, which is exactly what the search index caches. - c.markDirtyLocked(ctx, hashID) - return true, err + if !removed { + return nil + } + if len(kept) == 0 { + if _, err := tx.Delete(key); err != nil { + return err + } + if affectsHaystack(property) { + return refreshSearchTx(tx, hashID) + } + return nil + } + return writeFieldsTx(tx, hashID, map[string]string{property: strings.Join(kept, "|")}) + }) + return removed, err } -// FileTuple holds the (uuid, path, size, mtime, midhash) record for one -// file. Used by the watcher hydrator at startup to avoid re-mid-hashing -// files whose tuple matches what's already in Redis. -// -// HashID is the root identifier (UUIDv7 in the current model). Hydration -// uses HashID + (FilePath, Size, MtimeNano) to seed the in-memory state so -// the next scan can skip re-mid-hashing files whose size+mtime are unchanged. +// FileTuple holds the (uuid, path, size, mtime) record for one file. Used by +// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple +// matches what's already stored. type FileTuple struct { HashID string FilePath string @@ -621,453 +485,189 @@ type FileTuple struct { MtimeNano int64 } -// GetTuplesForAllFiles returns one FileTuple per indexed root, fetched in -// batched MGETs (no SCAN per root). Skips files missing any of filePath, -// sizeByte, or mtimeNano. Sub-second on local Redis for ~10K files. -// -// The midhash is intentionally NOT fetched here: it now lives only as a -// cids/ key-set member (no named field), and the skip-rehash decision -// is purely size+mtime — so reading it would cost a SCAN per file for no -// benefit. +// GetTuplesForAllFiles returns one FileTuple per indexed root, in batched +// multi-key reads. Skips files missing any of filePath, sizeByte, mtimeNano. func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) { - c.mu.RLock() - defer c.mu.RUnlock() - - if c.client == nil { - return nil, fmt.Errorf("not connected") - } - - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - - hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result() + s, err := c.kv() if err != nil { - return nil, fmt.Errorf("smembers index: %w", err) - } - if len(hashIDs) == 0 { - return nil, nil - } - - // Build the key list: 4 keys per root, in a fixed order so we can decode - // the MGET response by position. fields[i] for hashIDs[j] sits at - // keys[j*4+i]. - const fieldsPerFile = 3 - fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"} - keys := make([]string, 0, len(hashIDs)*fieldsPerFile) - for _, h := range hashIDs { - prefix := c.buildKeyPrefix(h) - for _, f := range fields { - keys = append(keys, prefix+f) - } - } - - // Chunked MGET — Redis caps args per command; 1000 is a safe batch. - const batch = 1000 - values := make([]interface{}, 0, len(keys)) - for start := 0; start < len(keys); start += batch { - end := start + batch - if end > len(keys) { - end = len(keys) - } - chunk, err := c.client.MGet(ctx, keys[start:end]...).Result() - if err != nil { - return nil, fmt.Errorf("mget chunk [%d:%d]: %w", start, end, err) - } - values = append(values, chunk...) - } - - out := make([]FileTuple, 0, len(hashIDs)) - for i, h := range hashIDs { - base := i * fieldsPerFile - filePath, _ := values[base].(string) - sizeStr, _ := values[base+1].(string) - mtimeStr, _ := values[base+2].(string) - if filePath == "" || sizeStr == "" || mtimeStr == "" { - continue - } - size, err := strconv.ParseInt(sizeStr, 10, 64) - if err != nil { - continue - } - mtime, err := strconv.ParseInt(mtimeStr, 10, 64) - if err != nil { - continue - } - out = append(out, FileTuple{ - HashID: h, - FilePath: filePath, - Size: size, - MtimeNano: mtime, - }) - } - return out, nil -} - -// SearchTuple is one record's search state: its hashID, a lowercased haystack of -// its searchable text fields (for token/substring match), and the full flat -// metadata map (so a search hit is returned straight from memory — no per-result -// Redis fetch). -type SearchTuple struct { - HashID string - Haystack string - Fields map[string]string -} - -// searchIndexFields are the metadata fields folded into a record's search -// haystack — must stay in sync with the fields the search handler documents. -var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"} - -// defaultIndexExcludePrefixes are field-name prefixes kept OUT of the in-memory -// search index. Redis still stores them and every per-record read still returns -// them — this narrows the *index*, never the store. -// -// `categories/newznab/` is the raw Newznab numeric category code, which -// METADATA_KEYS documents as a mirror of the canonical `categories/