replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

.gitea/workflows/docker-publish.yml

@@ -0,0 +1,87 @@
+# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
+# folder exists, and GitHub never reads it.
+#
+# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
+#   push  ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag>   docker login as metadec.eth with the
+#         IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
+#   pull  ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag>   anonymous
+# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
+# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
+# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
+# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
+# certificate without that CA, so the login is written straight into ~/.docker/config.json.
+# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
+# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
+
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag for the image (optional, defaults to sha)'
+        required: false
+        type: string
+
+env:
+  REGISTRY: ipcr-push-85-17-246-67.nip.io
+  IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Docker CLI
+        run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
+
+      - name: Trust the Mesh Router CA
+        run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
+        env:
+          IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+          buildkitd-config-inline: |
+            [registry."${{ env.REGISTRY }}"]
+              ca = ["/tmp/ipcr-ca.pem"]
+
+      - name: Log in to IPCR
+        env:
+          IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
+        run: |
+          mkdir -p ~/.docker
+          printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
+
+      - name: Extract metadata (tags, labels)
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -6,20 +6,20 @@
 # - Bash-based leader election script (flock on shared volume)
 # - meta-core Go binary (HTTP API, Mount API, Watcher)
 # - nginx (dashboard, WebDAV, API proxy)
-# - Redis (managed by supervisord on leader)
+# - SQLite metadata store, embedded in the Go binary (/meta-core/db/meta-core.db)
 # - rclone (single mount executor — SMB goes through rclone's :smb: backend)
 # - supervisord (process management - runs on leader only)
 #
 # Architecture:
 # - leader-election.sh handles flock, starts supervisord on leader
 # - Followers remain dormant, retry flock every 5s
-# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
+# - Only leader runs meta-core, nginx, rclone, mount-watcher
 # =============================================================================
 
 # -----------------------------------------------------------------------------
 # Stage 1: Build Go binary
 # -----------------------------------------------------------------------------
-FROM golang:1.21-alpine AS go-builder
+FROM golang:1.26-alpine AS go-builder
 
 RUN apk add --no-cache git
 
@@ -74,7 +74,7 @@ ARG RCLONE_VERSION=1.73.5
 # Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
 # all remote mounts go through rclone now.
 RUN apk add --no-cache \
-    redis \
+    sqlite \
     nginx \
     nginx-mod-http-dav-ext \
     supervisor \
@@ -104,7 +104,7 @@ RUN chmod +x /usr/local/bin/meta-core
 # Create directories
 RUN mkdir -p \
     /meta-core/locks \
-    /meta-core/db/redis \
+    /meta-core/db \
     /meta-core/services \
     /meta-core/mounts/errors \
     /files \
@@ -140,13 +140,12 @@ ENV META_CORE_PATH=/meta-core \
     SERVICE_VERSION=1.0.0 \
     META_CORE_HTTP_PORT=9000 \
     META_CORE_HTTP_HOST=0.0.0.0 \
-    REDIS_PORT=6379 \
     WATCH_INTERVAL_MS=1000
 
 # Expose ports
-# 80: nginx (dashboard + API proxy + WebDAV)
-# 6379: Redis (for other services)
-EXPOSE 80 6379
+# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port:
+# it is a file inside this container, reachable only through the HTTP API.
+EXPOSE 80
 
 # Health check
 HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \

README.md

@@ -1,39 +1,50 @@
 # meta-core
 
-Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
-metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
+Standalone Go service that owns the metadata store (an embedded SQLite file),
+the HTTP+SSE metadata surface, the
+User Data Layer store and signing keystore, and a WebDAV file server for the
+MetaMesh dev/prod stack.
 
 ## Overview
 
 meta-core runs as its own container (`metacore-app`). Other MetaMesh services
-(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
-reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
-is not exposed outside the container — all metadata I/O is mediated by the
-HTTP API.
+(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
+meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
+its `metamesh.core` advertise carries the full `/urls` payload) and then call its
+HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded
+in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API
+(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
 
 ### Key capabilities
 
 | Feature | Description |
 |---------|-------------|
-| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
-| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
-| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
-| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
+| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. |
+| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). |
+| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
+| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field` over (field name, value) of every record key, plus an FTS5 trigram table — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). |
+| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. |
 | **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
-| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
-| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
+| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
+| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
 | **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
 | **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
+| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
+| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
+| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
 
 ### Design characteristics
 
-- **Single-leader** — flock on a shared volume; no external consensus.
-- **HTTP-only externally** — Redis is bound to the container, never published.
+- **Single writer** — flock on the `/meta-core` volume; no external consensus.
+- **HTTP-only externally** — there is no database port; debug read-only with
+  `docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …`
+  (dev stacks: `/meta-core-db/meta-core.db`).
 - **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
-  on-disk Redis layout predates the current build (alpha clean-wipe policy;
+  on-disk store layout predates the current build (alpha clean-wipe policy;
   see `docs/uuid-rooted-metadata.md`).
-- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
-  image with `redis-server`, `rclone`, nginx, and supervisord.
+- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a
+  thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI
+  (for debugging).
 
 ## Architecture
 
@@ -41,34 +52,31 @@ HTTP API.
 ┌──────────────────────────────────────────────────────────────────────────┐
 │                    metacore-app  (standalone container)                  │
 │                                                                          │
+│  leader-election.sh  (flock /meta-core/locks/kv-leader.lock,             │
+│                       then exec supervisord)                             │
 │  supervisord                                                             │
-│   ├─ leader-election.sh  (flock /meta-core/locks/kv-leader.lock,         │
-│   │                       writes kv-leader.info, exec supervisord)      │
-│   ├─ redis-server        (AOF + RDB on /meta-core/db/redis)              │
-│   ├─ nginx               (TLS termination, proxy_cache for /webdav)      │
-│   ├─ rclone rcd          (mount daemon, RC API)                          │
+│   ├─ nginx  :80          (dashboard, /editor, API proxy,                 │
+│   │                       proxy_cache for /webdav, rclone GUI /rclone)   │
+│   ├─ rclone rcd          (mount daemon, RC API on :5572)                 │
 │   ├─ mount-watcher.sh    (rclone mount lifecycle)                        │
 │   └─ meta-core (Go)      ─►  HTTP API + SSE + WebDAV on :9000            │
+│                          ─►  embedded SQLite (db/meta-core.db)           │
+│                          ─►  UDP advertise/listen 239.255.99.1:9099      │
 └──────────────────────────────────────────────────────────────────────────┘
                                        │
                 ┌──────────────────────┴──────────────────────┐
-                │  /meta-core (shared volume)                 │
-                │   locks/kv-leader.{lock,info}               │
-                │   db/redis/                                 │
-                │   services/*.json                           │
+                │  /meta-core (meta-core's own volume)        │
+                │   locks/kv-leader.lock                      │
+                │   db/meta-core.db (+ -wal, -shm)            │
+                │   identity/accounts/*.json                  │
                 │   mounts/{mounts.json, errors/}             │
                 │   watchers.json                             │
                 │   cache/ (nginx proxy_cache)                │
                 └─────────────────────────────────────────────┘
-                                       ▲
-        ┌──────────────────────────────┼────────────────────────────────┐
-        │                              │                                │
-┌─────────────┐               ┌─────────────┐                ┌─────────────┐
-│  meta-sort  │               │  meta-fuse  │                │ meta-stremio│
-│             │               │             │                │             │
-│  reads      │  HTTP+SSE     │  reads      │  HTTP+SSE      │  reads      │
-│  leader.info├──────────────►│  leader.info├───────────────►│  leader.info│
-└─────────────┘               └─────────────┘                └─────────────┘
+
+   meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
+      │  1. hear the metamesh.core advertise on UDP multicast (carries /urls)
+      └─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume
 ```
 
 ## Quick start
@@ -82,8 +90,9 @@ make test            # run Go unit tests
 ```
 
 `make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
-`:local` tag is the one every sub-stack docker-compose.yml references
-(meta-share 10-peer federation, meta-gateway dev stacks); without it the
+`:local` tag is the one the sub-stack docker-compose files reference
+(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
+dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
 sub-stacks can't see a freshly-built image by name.
 
 > **Source-change → recreate.** `make docker` rebuilds and re-tags the
@@ -94,12 +103,14 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > make docker
-> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
->   up -d --force-recreate metagateway-generic-core
-> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
->   up -d --force-recreate metagateway-torznab-core
+> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
+>   up -d --force-recreate metagateway-hub-metacore
 > docker compose -f packages/meta-share/dev/docker-compose.yml \
 >   --profile share-10node up -d --force-recreate metashare-core metashare-core-test
+> docker compose -f packages/meta-read/dev/docker-compose.yml \
+>   up -d --force-recreate metaread-dev-core
+> docker compose -f packages/meta-listen/dev/docker-compose.yml \
+>   up -d --force-recreate metalisten-dev-core
 > ```
 >
 > The symptom of a missed recreate is missing API fields on `/urls`
@@ -109,7 +120,7 @@ sub-stacks can't see a freshly-built image by name.
 >
 > ```bash
 > docker inspect meta-core:local --format '{{.Id}}'                # fresh SHA
-> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
+> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'   # repeat for :latest
 > # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
 > ```
 
@@ -126,36 +137,43 @@ HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
 
 ## Configuration
 
-Environment variables consumed by `internal/config/config.go`:
+Environment variables (from `internal/config/config.go` unless noted):
 
 | Variable | Default | Description |
 |---|---|---|
-| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
+| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. |
 | `FILES_PATH` | `/files` | Files volume root. |
-| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
-| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
-| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
-| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
-| `REDIS_PORT` | `6379` | Local Redis port. |
+| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
+| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
+| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
+| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
+| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
+| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). |
+| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). |
+| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
+| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
 | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
-| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
-| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
-| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
-| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
-| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
-| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
-| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
-| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
+| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
+| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
+| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
+| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
+| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
 
-Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
-no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
-identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
-retry interval (`ELECTION_RETRY_SECS`, default `5`).
+`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
+`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
+`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
+them (leftovers of the file-based registry); setting them has no effect. There
+is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES`
+(gone with Redis and the in-memory search index; search results now include
+`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` /
+`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
+configs created via `/api/watchers`.
 
 ## API reference
 
-All endpoints below are served by `internal/api/server.go`. Examples assume
+All endpoints below are registered in `internal/api/server.go` (mounts,
+watcher and watchers routes in their own packages' `handlers.go`). Examples assume
 the in-container Go port `:9000`; from the host use
 `https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
 (debug-direct).
@@ -163,16 +181,47 @@ the in-container Go port `:9000`; from the host use
 ### Health, leader, URLs
 
 ```bash
-GET  /health           # storage + role
+GET  /health           # storage + role (`redis: bool` kept for compatibility = store open)
 GET  /status           # version, uptime, etc.
 GET  /leader           # current leader info (hostname, PID, timestamps)
-GET  /urls             # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
-GET  /services         # registered services + heartbeats
-GET  /api/services     # alias for /services
-GET  /services/{name}  # one service
-GET  /services/cleanup/stats
+GET  /urls             # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility)
+GET  /neighbors        # neighbours heard over UDP (beacon v2)
+GET  /api/neighbors    # same, under /api
+GET  /api/stats        # dashboard counters (see below)
 ```
 
+`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
+are **gone** — the registration-and-heartbeat registry they served was replaced
+by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
+with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
+than a 404; that is what made the dashboard fail with
+`SyntaxError: Unexpected token '<'` until 1.0.21.
+
+#### `GET /api/stats`
+
+Every headline counter in one request, split by cost:
+
+```jsonc
+{
+  "records": 88709,        // count of idx:roots/* — metadata roots, file or not
+  "redisKeys": 3115447,    // kv row count (name kept for compatibility; cached 60 s)
+  "redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility)
+  "identities": 3,         // signing accounts on disk
+  "files": { "count": 4864, "totalSize": 64587914424 },  // null until first sweep
+  "udlUsers": 5,           // accounts holding User Data Layer records
+  "sweptAt": 1789402401970,
+  "sweeping": false
+}
+```
+
+`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and
+`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
+box), so they are served from a single-flight cache refreshed in the background
+with a 60s TTL — the walk is **never** on the request path, because the
+dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this
+over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
+across records that share a file.
+
 ### Metadata — primary surface
 
 These are the routes other services use. `{hash}` may be a UUID root or
@@ -198,6 +247,8 @@ GET    /api/meta/{cid}                # resolve CID → metadata document
 GET    /api/file/{cid}/info           # CID resolution metadata
 GET    /file/{cid}                    # serve file bytes (range-aware)
 POST   /file/cid                      # compute CID for an uploaded file
+POST   /api/files/register        # mint + alias one file under /files/plugin by midhash
+                                  # (the watcher's path for one file; works with the watcher OFF)
 HEAD   /data/{hash}                   # existence + size
 GET    /data/{hash}/path              # absolute path on /files
 ```
@@ -222,10 +273,10 @@ PUT    /api/metadata/{hashId}/property
 
 GET    /api/kv/info
 GET    /api/kv/keys
-GET    /api/kv/tree
+GET    /api/kv/tree            # cursor is an opaque next-key ("0" when exhausted)
 GET    /api/kv/search
 GET    /api/kv/find
-GET    /api/kv/value
+GET    /api/kv/value           # type is always "string" for an existing key
 PUT    /api/kv/value
 DELETE /api/kv/value
 GET    /api/kv/key/{key...}
@@ -240,7 +291,8 @@ POST   /api/snapshot/wipe
 
 ### SSE event streams
 
-The HTTP-mirror of the Redis Streams. External services never read Redis
+The HTTP-mirror of the store's event streams (Redis Streams until 2026-10;
+IDs keep the `<ms>-<seq>` form). External services never read the store
 directly — they subscribe here with `Last-Event-ID` for resume. See
 [`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
 contract (one event per stream entry, opaque IDs, heartbeats, gap
@@ -249,6 +301,44 @@ signalling).
 ```bash
 GET /api/events/files     # file:events stream (watcher events)
 GET /api/events/meta      # meta:events stream (metadata mutations)
+GET /api/events/poll      # DEPRECATED long-poll over file:events — use SSE
+```
+
+### Identity (signing keystore)
+
+Reached through meta-search's identity proxy (and directly by meta-gateway,
+meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
+
+```bash
+GET    /api/identity               # status (never the private key)
+GET    /api/identity/accounts
+POST   /api/identity/generate      # the only call that returns a new private key
+POST   /api/identity/import
+POST   /api/identity/challenge     # one-shot challenge for proof-of-possession
+POST   /api/identity/reveal        # needs proof-of-possession
+DELETE /api/identity               # needs proof-of-possession; reports purged UDL counts
+POST   /api/identity/sign
+POST   /api/identity/sign-batch
+GET    /api/identity/aead-key
+```
+
+### User Data Layer
+
+Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
+writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
+cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec:<uid>/<cid>/<key>` → JSON `{record,version,ts,value?}`; index sets as one key per member).
+
+```bash
+GET  /api/udl/record?uid&cid&key
+PUT  /api/udl/record
+PUT  /api/udl/records                      # batch
+GET  /api/udl/user/{uid}/key/{key}
+GET  /api/udl/user/{uid}/cid/{cid}
+GET  /api/udl/user/{uid}/all
+GET  /api/udl/user/{uid}/count
+GET  /api/udl/cid/{cid}/users
+GET  /api/udl/cid/{cid}/aggregate          # meta-watch Aggregate wire shape
+GET  /api/udl/users/stats
 ```
 
 ### Mounts
@@ -284,46 +374,43 @@ The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
 return a deprecation pointer at the `/api/watchers/*` routes
 (`internal/watcher/handlers.go:32-33`).
 
-### Admin / WebDAV
+### WebDAV
 
 ```bash
-POST   /api/admin/migrate-dual-roots  # reunify stranded midhash-rooted entries
-POST   /api/admin/sweep-literature-echoes[?apply=true]
-       # strip query echoes off literature cards: anilistid on site cards,
-       # workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
-       # Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
 GET/PUT/DELETE /webdav/...            # mounted on /files
 ```
 
+The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`,
+`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were
+removed with the 2026-10 SQLite switch; they had been run once on
+watch.nsl.sh before the migration.
+
 No `/metrics` endpoint is exposed; observability is via container logs and
 the `/health` / `/status` JSON.
 
-## Leader gate
+## DB-file mutex
 
-Leader election is a bash flock loop in `docker/leader-election.sh`, not a
-Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
-script writes the leader API URL to `kv-leader.info` and `exec`s
-supervisord. When the process tree dies, the kernel releases the lock and
-any standby instance can take over. There is no follower path inside the
-Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
+`docker/leader-election.sh` is a bash flock loop, not a Go-level state
+machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
+script `exec`s supervisord; a second container mounting the same volume
+retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
+dies, the kernel releases the lock. There is no follower path inside the Go
+binary.
 
-`kv-leader.info` is a single plaintext line containing the API base URL:
-
-```
-http://10.0.1.50:9000
-```
-
-Other services read this file, then call `GET /urls` for full discovery.
-
-The dev compose runs a single `metacore-app`; the flock primitive is
-retained as a correctness anchor (one process owns Redis and the
-writable state under `/meta-core`), not as live failover.
+The lock is a **mutex, not service discovery** — it only stops two
+writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info`
+file it used to publish is gone: siblings locate meta-core over UDP
+(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
+`internal/leader` keeps its name for history, but only builds the `/urls`
+payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
+and the UDP announce share.
 
 ## Metadata storage shape (one-line summary)
 
-Roots are opaque UUIDv7 strings; every property is a separate Redis
-STRING at `file:<uuid>/<property>`; every known CID for a file is a
-reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
+Roots are opaque UUIDv7 strings; every property is a separate row
+of the `kv` table at `file:<uuid>/<property>`; every known CID is a key-set member
+`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
+(the canonical CID is derived by rank on read, never stored). Full details in
 [`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
 and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
 The authoritative registry of field semantics and value formats lives in
@@ -333,14 +420,16 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 
 | Package | Purpose |
 |---------|---------|
-| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
+| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. |
 | `internal/config` | Env-driven configuration, path helpers. |
-| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
-| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
-| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
-| `internal/discovery` | Service registry + dead-service cleaner. |
-| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
-| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
+| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
+| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
+| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). |
+| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field` + FTS (`search.go`), UDL store (`udl.go`, `udl_admin.go`), stats. |
+| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
+| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
+| `internal/events` | Boot-time republish of every field into `meta:events` (`meta_publisher.go`); live events are written by `internal/kv`. |
+| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. |
 | `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
 | `internal/snapshot` | Snapshot export / import / wipe. |
 | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
@@ -353,12 +442,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 ```
 1. Load configuration from environment
 2. Build LeaderInfoProvider (no election — the bash gate already won)
-3. Connect to local Redis (retry up to 30× / 1s)
-4. EnsureSchemaVersion — abort if the existing Redis layout is stale
-5. Start service discovery + dead-service cleaner
-6. Construct API server (initialises watcher, mounts, WebDAV)
-7. Start API server; on first connect, start MetaPublisher + schema Indexer
-   and republish existing metadata onto meta:events
+3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
+4. EnsureSchemaVersion — abort if the existing store layout is stale
+5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
+6. Construct API server (identity keystore migration, watcher, watchers,
+   mounts, WebDAV)
+7. Start API server + pollers; start MetaPublisher + schema Indexer, then
+   republish existing metadata onto meta:events in the background
+   (5k events per transaction)
 8. Wait for SIGINT/SIGTERM, shutdown in reverse order
 ```
 
@@ -367,13 +458,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
 | Dependency | Purpose |
 |------------|---------|
 | `github.com/gorilla/mux` | HTTP router |
-| `github.com/redis/go-redis/v9` | Redis client |
+| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) |
 | `github.com/google/uuid` | UUIDv7 minting |
+| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
 | `golang.org/x/net` | WebDAV support |
-| Go stdlib | context, net/http, os, sync, syscall |
 
-Runtime requirements: Go 1.21+ at build time; the container ships Redis,
-rclone, nginx, and supervisord alongside the binary.
+Runtime requirements: Go 1.21+ at build time; the container ships
+rclone, nginx, and supervisord alongside the binary, plus the React dashboard
+and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
 
 ## License
 

cmd/meta-core/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
@@ -10,8 +11,8 @@ import (
 
 	"github.com/metazla/meta-core/internal/api"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
@@ -29,51 +30,73 @@ func main() {
 	// Create leader info provider (Go binary only runs as leader - followers loop in bash)
 	leaderProvider := leader.NewLeaderInfoProvider(cfg)
 
-	// Connect to local Redis (supervisord starts Redis before meta-core)
-	// Retry connection since Redis may still be loading its dataset
-	redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
-	log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
-
-	storageClient := storage.NewClient("")
-	maxRetries := 30
-	retryDelay := 1 // seconds
-	var err error
-	for i := 0; i < maxRetries; i++ {
-		err = storageClient.Connect(redisURL)
-		if err == nil {
-			break
+	// Open the metadata store: an embedded SQLite file (internal/kv). There
+	// is no separate database process to wait for.
+	storageClient := storage.NewClient()
+	if err := storageClient.Connect(cfg.DBPath()); err != nil {
+		log.Fatalf("[meta-core] Failed to open the metadata store: %v", err)
 	}
-		log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err)
-		time.Sleep(time.Duration(retryDelay) * time.Second)
-	}
-	if err != nil {
-		log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err)
-	}
-	log.Println("[meta-core] Connected to Redis")
 
 	// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
-	// the existing Redis store predates the UUID-rooted schema, so legacy
-	// data can't silently leak into the new code paths. See
+	// the existing store predates the UUID-rooted schema, so legacy data
+	// can't silently leak into the new code paths. See
 	// docs/uuid-rooted-metadata.md.
 	if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
 		log.Fatalf("[meta-core] %v", err)
 	}
-	log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
+	log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion)
 
-	// Create and start service discovery
-	disc := discovery.NewService(cfg)
-	if err := disc.Start(); err != nil {
-		log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
+	// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
+	// as metamesh.core — carrying the /urls block services locate us by — plus
+	// its own metamesh.service/<name> nav entry. The payload is rebuilt from
+	// leaderProvider on every tick, which keeps it and GET /urls from drifting.
+	var mesh *meshdisco.Node
+	if cfg.EnableUDPDiscovery {
+		hostname, _ := os.Hostname()
+		mesh = meshdisco.New(meshdisco.Config{
+			Name:     cfg.ServiceName,
+			Instance: hostname,
+			Version:  Version,
+			Group:    cfg.DiscoveryGroup,
+			Port:     cfg.DiscoveryPort,
+			Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
+			Payload: func() (string, []meshdisco.Resource) {
+				service := meshdisco.Resource{
+					ID:        "service",
+					Caps:      []string{meshdisco.CapServicePrefix + cfg.ServiceName},
+					Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
 				}
-
-	// Create and start dead service cleaner
-	cleaner := discovery.NewCleaner(cfg)
-	if err := cleaner.Start(); err != nil {
-		log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
+				info := leaderProvider.LeaderInfo()
+				if info == nil {
+					return "starting", []meshdisco.Resource{service}
+				}
+				data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
+					Hostname:          info.Hostname,
+					BaseUrl:           info.BaseUrl,
+					ApiUrl:            info.ApiUrl,
+					WebdavUrl:         info.WebdavUrl,
+					WebdavUrlInternal: info.WebdavUrlInternal,
+				}})
+				core := meshdisco.Resource{
+					ID:        "core",
+					Caps:      []string{meshdisco.CapCore},
+					Endpoints: map[string]string{"http": info.ApiUrl},
+					Data:      data,
+				}
+				return "running", []meshdisco.Resource{core, service}
+			},
+		})
+		if err := mesh.Start(); err != nil {
+			log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
+			mesh = nil
+		}
+	} else {
+		log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
 	}
 
 	// Create and start API server
-	apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
+	apiServer := api.NewServer(cfg, leaderProvider, storageClient)
+	apiServer.SetMeshNode(mesh)
 	if err := apiServer.Start(); err != nil {
 		log.Fatalf("[meta-core] Failed to start API server: %v", err)
 	}
@@ -90,12 +113,10 @@ func main() {
 		log.Printf("[meta-core] Error stopping API server: %v", err)
 	}
 
-	if err := cleaner.Stop(); err != nil {
-		log.Printf("[meta-core] Error stopping service cleaner: %v", err)
+	if mesh != nil {
+		if err := mesh.Stop(); err != nil {
+			log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
 		}
-
-	if err := disc.Stop(); err != nil {
-		log.Printf("[meta-core] Error stopping service discovery: %v", err)
 	}
 
 	if err := storageClient.Close(); err != nil {
@@ -112,3 +133,17 @@ func waitForShutdown() {
 	<-sigChan
 }
 
+// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
+// the three-way fallback the file registry used (discovery/service.go):
+// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
+// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
+// the container IP.
+func coreBaseURL(cfg *config.Config) string {
+	if cfg.PublicURL != "" {
+		return cfg.PublicURL
+	}
+	if cfg.BaseURL != "" {
+		return cfg.BaseURL
+	}
+	return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
+}

dashboard/dist/index.html

@@ -21,7 +21,7 @@
         min-height: 100vh;
       }
     </style>
-    <script type="module" crossorigin src="/assets/index-BW9EQGCP.js"></script>
+    <script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
   </head>
   <body>

dashboard/src/components/ServiceNav.tsx

@@ -1,155 +1,53 @@
-import { useState, useEffect } from 'react';
+import { useEffect } from 'react';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
+// of defining the custom element.
+import '../meta-service-menu.js';
 
-interface ServiceInfo {
-  name: string;
-  hostname: string;
-  baseUrl: string;
-  status: string;
-  lastHeartbeat: string;
-}
+/**
+ * ServiceNav — the neighbouring meta-* services, as a burger menu.
+ *
+ * Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
+ * a framework-free custom element shared by every meta-* UI (see
+ * scripts/check-mirrors.sh). The old implementation was ~165 lines of React
+ * that existed in four near-identical copies and polled meta-core's
+ * /api/services; the element polls this service's own /api/neighbors instead,
+ * so the nav still renders when meta-core is down.
+ *
+ * The element lives in shadow DOM and cannot see this app's CSS variables, so
+ * the four --mm-nav-* tokens below map our palette onto it.
+ */
 
-interface ServicesResponse {
-  services: ServiceInfo[];
-  current: string;
-}
-
-const serviceIcons: Record<string, string> = {
-  'meta-core': '🗄️',
-  'meta-sort': '📁',
-  'meta-fuse': '🗂️',
-  'meta-stremio': '🎬',
-  'meta-orbit': '🌐',
-  'default': '📦'
-};
-
-function formatServiceName(name: string): string {
-  return name.split('-').map(word =>
-    word.charAt(0).toUpperCase() + word.slice(1)
-  ).join(' ');
+declare global {
+    // eslint-disable-next-line @typescript-eslint/no-namespace
+    namespace JSX {
+        interface IntrinsicElements {
+            'meta-service-menu': React.DetailedHTMLProps<
+                React.HTMLAttributes<HTMLElement>,
+                HTMLElement
+            > & { current?: string; endpoint?: string; label?: string };
+        }
+    }
 }
 
 function ServiceNav() {
-  const [services, setServices] = useState<ServiceInfo[]>([]);
-  const [loading, setLoading] = useState(true);
-  const currentService = 'meta-core';
-
     useEffect(() => {
-    const fetchServices = async () => {
-      try {
-        const response = await fetch('/services');
-        if (response.ok) {
-          const data: ServicesResponse = await response.json();
-          setServices(data.services || []);
-        }
-      } catch (error) {
-        console.error('Failed to fetch services:', error);
-      } finally {
-        setLoading(false);
-      }
-    };
-
-    fetchServices();
-    const interval = setInterval(fetchServices, 30000);
-    return () => clearInterval(interval);
+        // Import above is for the side effect of defining the element; this
+        // keeps bundlers from tree-shaking it away in production builds.
     }, []);
 
-  if (loading) return null;
-
-  const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
-
-  if (sortedServices.length === 0) return null;
-
     return (
-    <nav className="services-nav">
-      <span className="services-nav-label">Services:</span>
-      <div className="services-nav-items">
-        {sortedServices.map(service => {
-          const icon = serviceIcons[service.name] || serviceIcons.default;
-          const isActive = service.name === currentService;
-          return (
-            <a
-              key={service.name}
-              href={isActive ? '#' : service.baseUrl}
-              className={`service-link${isActive ? ' active' : ''}`}
-              onClick={(e) => {
-                e.preventDefault();
-                if (!isActive) {
-                  window.location.href = service.baseUrl;
-                }
-              }}
-            >
-              <span className="service-icon">{icon}</span>
-              <span>{formatServiceName(service.name)}</span>
-              <span className="service-status"></span>
-            </a>
-          );
-        })}
-      </div>
-
+        <>
+            <meta-service-menu current="meta-core" />
             <style>{`
-        .services-nav {
-          display: flex;
-          align-items: center;
-          gap: 0.5rem;
-        }
-
-        .services-nav-label {
-          color: #888;
-          font-size: 0.8rem;
-          white-space: nowrap;
-        }
-
-        .services-nav-items {
-          display: flex;
-          gap: 0.35rem;
-        }
-
-        .service-link {
-          display: inline-flex;
-          align-items: center;
-          gap: 0.3rem;
-          padding: 0.35rem 0.6rem;
-          border-radius: 6px;
-          text-decoration: none;
-          font-size: 0.8rem;
-          font-weight: 500;
-          transition: all 0.2s;
-          color: #ccc;
-          background: #0f3460;
-          border: 1px solid #1a4a7a;
-        }
-
-        .service-link:hover {
-          background: #1a4a7a;
-          border-color: rgba(78, 205, 196, 0.5);
-          text-decoration: none;
-          color: #fff;
-        }
-
-        .service-link.active {
-          background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
-          border-color: rgba(78, 205, 196, 0.5);
-          color: #fff;
-        }
-
-        .service-link .service-icon {
-          font-size: 0.9rem;
-        }
-
-        .service-link .service-status {
-          width: 5px;
-          height: 5px;
-          border-radius: 50%;
-          background: #4ecdc4;
-        }
-
-        @media (max-width: 800px) {
-          .services-nav-label {
-            display: none;
-          }
+                meta-service-menu {
+                    --mm-nav-fg: var(--text-primary, #e0e0e0);
+                    --mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
+                    --mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
+                    --mm-nav-accent: var(--accent-primary, #4ecdc4);
                 }
             `}</style>
-    </nav>
+        </>
     );
 }
 

dashboard/src/lib/api.ts

@@ -0,0 +1,28 @@
+// JSON fetch that fails with a sentence instead of a parser error.
+//
+// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
+// path it does not have a proxy rule for answers 200 text/html with the SPA
+// shell. A caller that does `res.ok && res.json()` on such a path gets
+// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
+// no clue which request produced it — which is exactly how a route deleted
+// from the Go router (`/services`, removed in favour of /api/neighbors) sat
+// broken on the dashboard.
+//
+// So: check the content type, and name the URL in every error.
+
+export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
+  const res = await fetch(url, init);
+  const ct = res.headers.get('content-type') || '';
+
+  if (!res.ok) {
+    throw new Error(`${url} → HTTP ${res.status}`);
+  }
+  if (!ct.includes('application/json')) {
+    // The SPA fallback is the overwhelmingly likely cause; say so.
+    throw new Error(
+      `${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
+        `missing and nginx served the dashboard shell instead)`
+    );
+  }
+  return (await res.json()) as T;
+}

dashboard/src/meta-service-menu.js

@@ -0,0 +1,308 @@
+/**
+ * <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
+ *
+ * Replaces the eight divergent "services bar" implementations (four copies of
+ * a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
+ * ones) with a single framework-free custom element. It drops into the React
+ * dashboards as a plain tag — no React dependency — and into the vanilla UIs
+ * as an ES module.
+ *
+ * Data comes from the host service's own `GET /api/neighbors`, same-origin.
+ * Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
+ * make a fetch to a neighbour a dead end. Links are plain navigations.
+ *
+ * ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
+ * `scripts/check-mirrors.sh`. Edit one → edit all.
+ *
+ * Shadow DOM is not optional here. The eight UIs use four incompatible token
+ * vocabularies, and two of them define *different colours under the same
+ * names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
+ * #161b22 in meta-dup). meta-search and meta-share are also the only
+ * light-mode-capable UIs, so a hardcoded dark palette would break them.
+ * The element therefore styles itself from four tokens with dark defaults:
+ *
+ *     --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
+ *
+ * Each host maps its own variables onto those, e.g.
+ *     meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
+ *
+ * Attributes:
+ *   current   name of the host service, so it can be marked and not linked
+ *   endpoint  override the default "/api/neighbors"
+ *   label     button tooltip (default "Services")
+ *
+ * Property:
+ *   rewriteUrl  optional (name, url, neighbour) => url, for hosts that must
+ *               translate internal hostnames into browser-reachable ones
+ *               (meta-gateway does this for its dev/prod split).
+ */
+
+const ICONS = {
+  "meta-core": "⚙️",
+  "meta-sort": "📁",
+  "meta-fuse": "🗂️",
+  "meta-stremio": "🎬",
+  "meta-dup": "🔍",
+  "meta-search": "🔎",
+  "meta-share": "🔗",
+  "meta-gateway": "🌉",
+  "meta-watch": "📺",
+  "meta-listen": "🎧",
+  "meta-read": "📖",
+};
+const DEFAULT_ICON = "📦";
+const POLL_MS = 30000;
+
+function titleCase(name) {
+  return String(name || "")
+    .split("-")
+    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+    .join(" ");
+}
+
+const TEMPLATE = `
+<style>
+  :host {
+    --_fg: var(--mm-nav-fg, #e8eaed);
+    --_bg: var(--mm-nav-bg, #1a1a2e);
+    --_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
+    --_accent: var(--mm-nav-accent, #4ecdc4);
+    position: relative;
+    display: inline-block;
+    font-family: inherit;
+    font-size: 14px;
+    line-height: 1.4;
+  }
+  button.trigger {
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    gap: 0.4rem;
+    width: 2.1rem;
+    height: 2.1rem;
+    padding: 0;
+    cursor: pointer;
+    color: var(--_fg);
+    background: transparent;
+    border: 1px solid var(--_border);
+    border-radius: 8px;
+    font: inherit;
+    transition: border-color 0.15s, background 0.15s;
+  }
+  button.trigger:hover { border-color: var(--_accent); }
+  button.trigger:focus-visible {
+    outline: 2px solid var(--_accent);
+    outline-offset: 2px;
+  }
+  button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
+  .bars { display: block; width: 15px; }
+  .bars span {
+    display: block;
+    height: 2px;
+    margin: 3px 0;
+    border-radius: 2px;
+    background: currentColor;
+  }
+  .panel {
+    position: absolute;
+    top: calc(100% + 6px);
+    right: 0;
+    z-index: 1000;
+    min-width: 13rem;
+    max-height: 70vh;
+    overflow-y: auto;
+    padding: 0.3rem;
+    background: var(--_bg);
+    border: 1px solid var(--_border);
+    border-radius: 10px;
+    box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
+  }
+  .panel[hidden] { display: none; }
+  a.item, .item {
+    display: flex;
+    align-items: center;
+    gap: 0.55rem;
+    padding: 0.45rem 0.6rem;
+    border-radius: 6px;
+    color: var(--_fg);
+    text-decoration: none;
+    white-space: nowrap;
+  }
+  a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
+  .item.active { color: var(--_accent); font-weight: 600; cursor: default; }
+  .item.dead { opacity: 0.55; cursor: not-allowed; }
+  .icon { font-size: 1rem; line-height: 1; }
+  .dot {
+    width: 6px;
+    height: 6px;
+    margin-left: auto;
+    border-radius: 50%;
+    background: var(--_accent);
+  }
+  .empty {
+    padding: 0.6rem;
+    opacity: 0.7;
+    font-size: 0.85em;
+    white-space: nowrap;
+  }
+  /* Respect a host that opts out of motion. */
+  @media (prefers-reduced-motion: reduce) {
+    button.trigger { transition: none; }
+  }
+</style>
+<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
+  <span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
+</button>
+<div class="panel" role="menu" hidden></div>
+`;
+
+class MetaServiceMenu extends HTMLElement {
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+    this.shadowRoot.innerHTML = TEMPLATE;
+    this._btn = this.shadowRoot.querySelector("button.trigger");
+    this._panel = this.shadowRoot.querySelector(".panel");
+    this._neighbors = [];
+    this._timer = null;
+    this._open = false;
+
+    this._onDocClick = (e) => {
+      if (!this._open) return;
+      if (e.composedPath().includes(this)) return;
+      this._setOpen(false);
+    };
+    this._onKey = (e) => {
+      if (e.key === "Escape" && this._open) {
+        this._setOpen(false);
+        this._btn.focus();
+      }
+    };
+  }
+
+  connectedCallback() {
+    this._btn.title = this.getAttribute("label") || "Services";
+    this._btn.setAttribute("aria-label", this._btn.title);
+    this._btn.addEventListener("click", () => this._setOpen(!this._open));
+    document.addEventListener("click", this._onDocClick);
+    document.addEventListener("keydown", this._onKey);
+
+    this.refresh();
+    this._timer = setInterval(() => this.refresh(), POLL_MS);
+  }
+
+  disconnectedCallback() {
+    if (this._timer) clearInterval(this._timer);
+    this._timer = null;
+    document.removeEventListener("click", this._onDocClick);
+    document.removeEventListener("keydown", this._onKey);
+  }
+
+  get endpoint() {
+    return this.getAttribute("endpoint") || "/api/neighbors";
+  }
+
+  get current() {
+    return this.getAttribute("current") || "";
+  }
+
+  async refresh() {
+    try {
+      const res = await fetch(this.endpoint, {
+        headers: { Accept: "application/json" },
+      });
+      if (!res.ok) throw new Error(String(res.status));
+      const data = await res.json();
+      // `services` is the legacy alias some backends still emit.
+      this._neighbors = data.neighbors || data.services || [];
+    } catch {
+      // Silent: the menu is navigation, not a health indicator. Keep whatever
+      // we last had rather than blanking the list on one failed poll.
+    }
+    this._render();
+  }
+
+  _setOpen(open) {
+    this._open = open;
+    this._panel.hidden = !open;
+    this._btn.setAttribute("aria-expanded", String(open));
+    if (open) this.refresh();
+  }
+
+  _render() {
+    const panel = this._panel;
+    panel.textContent = "";
+
+    const seen = new Set();
+    const rows = [];
+    for (const n of this._neighbors) {
+      const name = n.name || "";
+      if (!name || seen.has(name)) continue;
+      // Beacon v2 neighbours include plugins; the nav lists services only.
+      // (A row without `caps` comes from a pre-v2 backend — keep it.)
+      const isService = (c) => String(c).startsWith("metamesh.service/");
+      if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
+      seen.add(name);
+      rows.push(n);
+    }
+    // Always show the host itself, even before its own echo comes back.
+    if (this.current && !seen.has(this.current)) {
+      rows.push({ name: this.current, baseUrl: "" });
+    }
+    rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
+
+    if (rows.length === 0) {
+      const empty = document.createElement("div");
+      empty.className = "empty";
+      empty.textContent = "No services found";
+      panel.appendChild(empty);
+      return;
+    }
+
+    for (const n of rows) {
+      const isActive = n.name === this.current;
+      let url = n.baseUrl || "";
+      if (typeof this.rewriteUrl === "function") {
+        try {
+          url = this.rewriteUrl(n.name, url, n) || "";
+        } catch {
+          /* a bad hook must not blank the menu */
+        }
+      }
+
+      // A neighbour with no reachable URL is shown but not linked — better
+      // than a link that goes nowhere.
+      const el = document.createElement(isActive || !url ? "span" : "a");
+      el.className =
+        "item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
+      if (el.tagName === "A") {
+        el.href = url;
+        el.setAttribute("role", "menuitem");
+      }
+      if (!isActive && !url) el.title = "No reachable URL advertised";
+
+      const icon = document.createElement("span");
+      icon.className = "icon";
+      icon.textContent = ICONS[n.name] || DEFAULT_ICON;
+
+      const label = document.createElement("span");
+      label.textContent = titleCase(n.name);
+
+      el.append(icon, label);
+      // Beacon v2 omits `status` when running.
+      if ((n.status || "running") === "running" || isActive) {
+        const dot = document.createElement("span");
+        dot.className = "dot";
+        dot.title = n.status || "running";
+        el.appendChild(dot);
+      }
+      panel.appendChild(el);
+    }
+  }
+}
+
+if (!customElements.get("meta-service-menu")) {
+  customElements.define("meta-service-menu", MetaServiceMenu);
+}
+
+export default MetaServiceMenu;

dashboard/src/pages/Dashboard.tsx

@@ -1,4 +1,5 @@
 import { useEffect, useState } from 'react';
+import { getJSON } from '../lib/api';
 
 interface HealthStatus {
   status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
   timestamp: string;
 }
 
+interface FileStats {
+  count: number;
+  totalSize: number;
+}
+
+interface Stats {
+  records: number;
+  redisKeys: number;
+  redisMemory: string;
+  identities: number;
+  files: FileStats | null;
+  udlUsers: number | null;
+  sweptAt: number;
+  sweeping: boolean;
+}
+
 interface WatcherStatus {
   id: string;
   path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
   count: number;
 }
 
-interface ServiceInfo {
-  name: string;
-  api: string;
-  capabilities: string[];
-  timestamp: number;
-}
-
 const cardStyle: React.CSSProperties = {
   background: '#16213e',
   borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
 
 const gridStyle: React.CSSProperties = {
   display: 'grid',
-  gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
+  // 200px, not 220: five tiles must fit one row inside the 1200px container
+  // (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
+  gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
   gap: '1rem',
+  marginBottom: '1rem',
 };
 
 const statStyle: React.CSSProperties = {
   fontSize: '2rem',
   fontWeight: 'bold',
   color: '#4ade80',
+  lineHeight: 1.2,
 };
 
+const labelStyle: React.CSSProperties = {
+  marginBottom: '0.75rem',
+  color: '#888',
+  fontSize: '0.85rem',
+  textTransform: 'uppercase',
+  letterSpacing: '0.04em',
+};
+
+const noteStyle: React.CSSProperties = {
+  color: '#6b7a99',
+  fontSize: '0.8rem',
+  marginTop: '0.4rem',
+};
+
+const numberFmt = new Intl.NumberFormat();
+
+function formatCount(n: number | null | undefined): string {
+  return n === null || n === undefined ? '—' : numberFmt.format(n);
+}
+
+// Binary units: this counts bytes on disk, and every other tool the operator
+// compares against (du, docker, the mount page) reports GiB.
+function formatBytes(bytes: number | null | undefined): string {
+  if (bytes === null || bytes === undefined) return '—';
+  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
+}
+
+function Tile({
+  label,
+  value,
+  note,
+  pending,
+}: {
+  label: string;
+  value: string;
+  note?: string;
+  pending?: boolean;
+}) {
+  return (
+    <div style={cardStyle}>
+      <div style={labelStyle}>{label}</div>
+      <div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
+        {value}
+      </div>
+      {note && <div style={noteStyle}>{note}</div>}
+    </div>
+  );
+}
+
 export default function Dashboard() {
   const [health, setHealth] = useState<HealthStatus | null>(null);
+  const [stats, setStats] = useState<Stats | null>(null);
   const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
-  const [services, setServices] = useState<ServiceInfo[]>([]);
   const [error, setError] = useState<string | null>(null);
 
   useEffect(() => {
+    let cancelled = false;
+
     const fetchData = async () => {
-      try {
-        // Fetch health
-        const healthRes = await fetch('/health');
-        if (healthRes.ok) {
-          setHealth(await healthRes.json());
-        }
+      // Settled, not all: one dead endpoint must not blank every tile.
+      const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
+        getJSON<HealthStatus>('/health'),
+        getJSON<Stats>('/api/stats'),
+        getJSON<WatchersResponse>('/api/watchers'),
+      ]);
+      if (cancelled) return;
 
-        // Fetch watchers status
-        const watchersRes = await fetch('/api/watchers');
-        if (watchersRes.ok) {
-          setWatchersData(await watchersRes.json());
-        }
+      if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
+      if (statsRes.status === 'fulfilled') setStats(statsRes.value);
+      if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
 
-        // Fetch services
-        const servicesRes = await fetch('/services');
-        if (servicesRes.ok) {
-          const data = await servicesRes.json();
-          setServices(data.services || []);
-        }
-
-        setError(null);
-      } catch (err) {
-        setError(String(err));
-      }
+      const failures = [healthRes, statsRes, watchersRes]
+        .filter((r): r is PromiseRejectedResult => r.status === 'rejected')
+        .map((r) => String(r.reason?.message ?? r.reason));
+      setError(failures.length ? failures.join(' · ') : null);
     };
 
     fetchData();
     const interval = setInterval(fetchData, 5000);
-    return () => clearInterval(interval);
+    return () => {
+      cancelled = true;
+      clearInterval(interval);
+    };
   }, []);
 
   const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
     }
   };
 
-  // Aggregate watcher stats
-  const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
-  const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
-  const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
-  const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
+  const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
+  const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
+  const lastScan =
+    watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
+
+  // Records we hold metadata for but have no local file: gateway/cid-rooted
+  // entries, plus anything whose file went away. The interesting number on a
+  // client box, where it should be most of them.
+  const fileless =
+    stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
+
+  const sweepPending = !!stats && !stats.files;
+  const sweptNote = (() => {
+    if (!stats) return undefined;
+    if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
+    if (stats.sweeping) return 'refreshing…';
+    if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
+    return undefined;
+  })();
 
   return (
-    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
+    <div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
       <h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
 
       {error && (
         <div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
-          Error: {error}
+          {error}
         </div>
       )}
 
       <div style={gridStyle}>
-        {/* Health Card */}
+        <Tile
+          label="Files discovered"
+          value={formatCount(stats?.files?.count)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Library size"
+          value={formatBytes(stats?.files?.totalSize)}
+          note={sweptNote}
+          pending={sweepPending}
+        />
+        <Tile
+          label="Metadata entries"
+          value={formatCount(stats?.records)}
+          note={
+            fileless === null
+              ? 'records in the index'
+              : `${formatCount(fileless)} with no local file`
+          }
+        />
+        <Tile
+          label="Store keys"
+          value={formatCount(stats?.redisKeys)}
+          note={stats?.redisMemory ? `${stats.redisMemory} on disk` : undefined}
+        />
+        <Tile
+          label="Identities"
+          value={formatCount(stats?.identities)}
+          note={
+            stats?.udlUsers !== null && stats?.udlUsers !== undefined
+              ? `${formatCount(stats.udlUsers)} with user data`
+              : 'signing accounts'
+          }
+        />
+      </div>
+
+      <div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
+          <div style={labelStyle}>Service health</div>
           {health ? (
             <>
               <div style={statStyle}>
                 {health.status === 'ok' ? 'Healthy' : health.status}
               </div>
-              <p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
+              <p style={noteStyle}>
+                Metadata store: {health.redis ? 'Open' : 'Unavailable'}
+              </p>
             </>
           ) : (
-            <p>Loading...</p>
+            <p style={noteStyle}>Loading…</p>
           )}
         </div>
 
-        {/* Watchers Status Card */}
         <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
+          <div style={labelStyle}>File watchers</div>
           {watchersData ? (
             <>
-              <div style={statStyle}>{totalFiles}</div>
-              <p>Files discovered</p>
-              <p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
-              <p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
+              <div style={statStyle}>
+                {activeWatchers} / {watchersData.count}
+              </div>
+              <p style={noteStyle}>active watchers</p>
+              <p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
               {lastScan > 0 && (
-                <p>Last scan: {new Date(lastScan).toLocaleString()}</p>
+                <p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
               )}
               <button
                 onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
               </button>
             </>
           ) : (
-            <p>Loading...</p>
-          )}
-        </div>
-
-        {/* Services Card */}
-        <div style={cardStyle}>
-          <h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
-          <div style={statStyle}>{services.length}</div>
-          <p>Services registered</p>
-          {services.length > 0 && (
-            <ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
-              {services.map((svc) => (
-                <li key={svc.name} style={{ marginBottom: '0.5rem' }}>
-                  <strong>{svc.name}</strong>
-                  <br />
-                  <span style={{ color: '#888', fontSize: '0.9rem' }}>
-                    {svc.capabilities?.join(', ') || 'No capabilities'}
-                  </span>
-                </li>
-              ))}
-            </ul>
+            <p style={noteStyle}>Loading…</p>
           )}
         </div>
       </div>

dashboard/src/pages/Identity.tsx

@@ -179,7 +179,7 @@ async function fetchAccounts(): Promise<Account[]> {
 // Resolve an account's display name from the User Data Layer, or null.
 //
 // Never throws and never reports an error upward: a name is decoration on top
-// of the uid, and the account list must render identically whether Redis is up
+// of the uid, and the account list must render identically whether the store is up
 // (identity itself is file-backed, the UDL is not). The name is also *unsigned*
 // — meta-core stores the plaintext beside the signed record and verifies
 // nothing — so the uid stays on screen as the real identity, and the length cap
@@ -209,7 +209,7 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
 // so per-row calls would walk the keyspace once per account.
 //
 // Never throws, for the same reason fetchDisplayName does not: the counts are
-// context on top of the uid, and the list must render identically when Redis is
+// context on top of the uid, and the list must render identically when the store is
 // down — accounts are file-backed, their data is not.
 //
 // Returns null on failure rather than an empty map. The difference is
@@ -603,7 +603,7 @@ function AccountCard({
                       records === 1 ? 'record' : 'records'
                     } — likes, ratings, watch progress across ${titles.toLocaleString()} ${
                       titles === 1 ? 'title' : 'titles'
-                    } — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
+                    } — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '}
               This cannot be undone and there is no backup.
             </li>
             <li>
@@ -655,12 +655,12 @@ export default function Identity() {
       return;
     }
     if (gen !== reqGen.current) return;
-    // Render the list before resolving names, so a down Redis delays nothing.
+    // Render the list before resolving names, so a down store delays nothing.
     setAccounts(list);
     setLoading(false);
 
     // Names and counts both come after the list is on screen, and neither can
-    // fail the page: a down Redis costs decoration, not the account list.
+    // fail the page: a down store costs decoration, not the account list.
     const [settled, counts] = await Promise.all([
       Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
       fetchStats(),

docker/entrypoint.sh

@@ -9,8 +9,7 @@ set -e
 
 # Create required directories
 mkdir -p /meta-core/locks
-mkdir -p /meta-core/db/redis
-mkdir -p /meta-core/services
+mkdir -p /meta-core/db
 mkdir -p /meta-core/mounts/errors
 mkdir -p /files
 mkdir -p /var/log/supervisor

docker/leader-election.sh

@@ -2,9 +2,14 @@
 # =============================================================================
 # Leader Election Script for meta-core
 #
-# Uses flock for leader election. Only the leader runs the full application
-# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
-# Followers remain dormant, periodically retrying to acquire the lock.
+# The flock here is a MUTEX, not service discovery. It is the only thing
+# stopping two containers that mount the same /meta-core volume from putting
+# two writers on one database file (/meta-core/db/meta-core.db). Sibling
+# services no longer read anything from this directory — they locate meta-core
+# over UDP (beacon v2, see docs/project-architecture/beacon-v2.md), so the
+# kv-leader.info file
+# this script used to publish is gone. Keep the lock; it costs nothing and the
+# corruption it prevents is unrecoverable.
 #
 # Environment variables:
 #   META_CORE_PATH         - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +25,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
 ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
 
 LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
-INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
 SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
 
 # Get the container's IP address
@@ -40,35 +44,13 @@ get_local_ip() {
     echo "$ip"
 }
 
-# Write leader info file (plain text API URL)
-write_leader_info() {
-    local ip=$(get_local_ip)
-    local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
-
-    # Write atomically using temp file + rename
-    local temp_file="${INFO_FILE}.tmp"
-    echo "$api_url" > "$temp_file"
-    mv "$temp_file" "$INFO_FILE"
-
-    echo "[election] Wrote leader info: $api_url"
-}
-
-# Clean up leader info file on shutdown
-cleanup() {
-    echo "[election] Cleaning up..."
-    rm -f "$INFO_FILE"
-    exit 0
-}
-
 # Set up signal handlers for graceful shutdown
-trap cleanup SIGTERM SIGINT
 
 # Ensure lock directory exists
 mkdir -p "$(dirname "$LOCK_FILE")"
 
 echo "[election] Starting leader election..."
 echo "[election] Lock file: $LOCK_FILE"
-echo "[election] Info file: $INFO_FILE"
 
 # Open lock file for flock (file descriptor 200)
 exec 200>"$LOCK_FILE"
@@ -79,9 +61,6 @@ while true; do
     if flock -n 200; then
         echo "[election] Acquired flock - transitioning to LEADER"
 
-        # Write leader info before starting services
-        write_leader_info
-
         # exec supervisord - replaces this process
         # When supervisord exits, the flock is released automatically
         echo "[election] Starting supervisord..."

docker/nginx.conf

@@ -123,14 +123,10 @@ http {
         # =========================================================================
         # Service discovery
         # =========================================================================
-        location /services {
-            proxy_pass http://meta-core-api/services;
-            proxy_http_version 1.1;
-            proxy_set_header Host $host;
-        }
-
-        location /services/ {
-            proxy_pass http://meta-core-api/services/;
+        # meta-discovery v1 neighbours (supersedes /services, which stays
+        # while consumers migrate).
+        location /neighbors {
+            proxy_pass http://meta-core-api/neighbors;
             proxy_http_version 1.1;
             proxy_set_header Host $host;
         }

docker/supervisord.conf

@@ -16,26 +16,13 @@ serverurl=unix:///var/run/supervisor.sock
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 # =============================================================================
-# Redis: Key-value store (managed by supervisord, started before meta-core)
-# Previously spawned by Go code, now managed by bash-based leader election
-# =============================================================================
-[program:redis]
-command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
-priority=1
-autostart=true
-autorestart=true
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-startsecs=2
-
-# =============================================================================
-# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
+# meta-core: HTTP API, Mount API, File Watcher, metadata store (embedded SQLite)
 # =============================================================================
 [program:meta-core]
 command=/usr/local/bin/meta-core
 priority=10
+; Room to finish in-flight writes and checkpoint the WAL on shutdown.
+stopwaitsecs=60
 autostart=true
 autorestart=true
 stdout_logfile=/dev/stdout
@@ -50,7 +37,6 @@ environment=
     SERVICE_VERSION="%(ENV_SERVICE_VERSION)s",
     META_CORE_HTTP_PORT="%(ENV_META_CORE_HTTP_PORT)s",
     META_CORE_HTTP_HOST="%(ENV_META_CORE_HTTP_HOST)s",
-    REDIS_PORT="%(ENV_REDIS_PORT)s",
     WATCH_INTERVAL_MS="%(ENV_WATCH_INTERVAL_MS)s",
     API_PORT="80",
     BASE_URL="%(ENV_BASE_URL)s"

docs/api-mediated-access.md

@@ -9,6 +9,16 @@ on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
 meta-core is the single writer to the metadata keyspace — an assumption
 that's now actually enforced rather than aspirational.
 
+> **2026-10: meta-core no longer uses Redis at all.** Its store is now an
+> embedded SQLite file (`/meta-core/db/meta-core.db`, see
+> [metadata-storage-structure.md](metadata-storage-structure.md)) and the
+> `file:events` / `meta:events` streams are rows of that store. The lockdown
+> below is now structural — there is no database port anywhere — and the
+> SSE wire contract was kept byte-for-byte, so no consumer changed. The
+> rest of this doc is the design record of the Redis-era lockdown; the
+> sections that describe *current* behaviour (SSE retention, reconnect,
+> error handling) have been updated in place.
+
 See "Implementation outcome" at the bottom for the delta between this doc's
 plan and what shipped (notably the write-path CID resolution and the
 dual-root migration sweep, which weren't in the original plan but turned
@@ -85,7 +95,7 @@ we don't fully control.
 ```
 
 **The single-writer rule.** meta-core has exclusive write authority over
-the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
+the `file:*`, `cid:*`, `file:__index__` (now `idx:*`), and `meta-core:*` keyspaces. No
 other service can write — not because of an honoured convention, but
 because they can't reach Redis at all.
 
@@ -243,6 +253,25 @@ After PR A, the cursor contract is **"durable across meta-core restart
 and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
 when they persist a Last-Event-ID.
 
+**Current backing store (since 2026-10).** The streams are rows of
+meta-core's SQLite kv table — `ev:<stream>/<ms13>-<seq10>` → JSON of the
+entry's fields, plus a high-water mark `evhwm:<stream>` that seeds the next
+ID (so a `ClearStream` reset never reissues an ID an old cursor treats as
+seen). The contract above holds unchanged:
+
+- Entries persist across restart (they are in the database file).
+- Retention is ~100k entries per stream, trimmed every 10k appends.
+- IDs keep the Redis `<ms>-<seq>` form, so cursors persisted by meta-sort /
+  meta-share / meta-stremio before the switch still compare.
+- `meta:events` entries (`{type: set|del, key, ts}`) are appended by the
+  storage engine **in the same transaction** as the `file:` key change —
+  `del` only when the key existed, `set` even for an unchanged value. The
+  Redis keyspace-notification hop (PSUBSCRIBE in `MetaPublisher`), which
+  could drop events when it lagged, is gone.
+- The boot-time republish of every field into `meta:events` is kept
+  (meta-fuse rebuilds its VFS only by replaying from `0-0`), batched 5k
+  events per transaction.
+
 ### Heartbeats
 
 Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
@@ -254,10 +283,13 @@ per the SSE spec.
 
 Per the SSE spec the client (or `EventSource`) auto-reconnects after a
 network error, sending `Last-Event-ID`. The server resumes from the next
-entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
+entry after that ID (a range read of the stream's rows; under Redis this
+was `XREAD … STREAMS file:events <id>`). With no `Last-Event-ID` the stream
+starts at the head, resolved once at connect; `0` / `0-0` go through the
+gap path and replay from the beginning.
 
-**Gap handling.** If the requested ID has been trimmed out of the Redis
-Stream by retention, the server emits one synthetic event before resuming
+**Gap handling.** If the requested ID is older than the oldest retained
+entry, the server emits one synthetic event before resuming
 from the oldest available entry:
 
 ```
@@ -271,9 +303,10 @@ react. Consumers that don't care ignore it.
 
 ### Error handling
 
-- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
-- **Redis fails mid-stream**: close the SSE connection. Client reconnects
-  with `Last-Event-ID`; the cycle resumes when Redis recovers.
+- **Store unavailable on connect**: 503 with `Retry-After: 5`.
+- **Store read fails mid-stream**: close the SSE connection. Client
+  reconnects with `Last-Event-ID`; the cycle resumes when meta-core is back
+  (a meta-core restart is the common case — SSE clients just reconnect).
 - **Client disconnects**: handler goroutine exits cleanly; no per-client
   state on meta-core.
 
@@ -307,7 +340,8 @@ inside-only until a concrete external consumer appears.
 ### Implementation footprint
 
 The SSE handler itself is ~80 lines of Go in meta-core: one handler per
-stream, each running an `XREAD … BLOCK 30s` loop and writing
+stream, each running a read-after-cursor loop (an `XREAD … BLOCK 30s` loop
+in the Redis era) and writing
 SSE-formatted lines to the response. Go's `net/http` natively supports
 streaming responses with `http.Flusher.Flush()`. Standard SSE clients
 exist for every language we use: Node `eventsource`, Rust
@@ -562,11 +596,11 @@ metacore-app produces a connection refused. The lock is real.
 | Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
 | Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
 | Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
-| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
+| Backing store for streams? | Redis Streams at the time; SQLite kv rows since 2026-10 | SSE wraps it; the 2026-10 swap proved it — no consumer changed |
 | Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
 | Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
 | **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
-| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
+| **SSE event-id contract?** | **Opaque to consumers** | It is `<ms>-<seq>` (the Redis Stream entry form, kept after the 2026-10 SQLite switch); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
 | **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
 | **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
 | **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
@@ -591,7 +625,7 @@ Stable `error` slugs the client should switch on:
 | `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
 | `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
 | `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
-| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
+| `storage_unavailable` | 503 | yes | Store transient (Redis at the time; the embedded SQLite store since 2026-10). Caller should retry with backoff. |
 | `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
 
 **Implemented.** Every 4xx/5xx response across `internal/api`,
@@ -709,7 +743,9 @@ contract violation."
    One-shot fixer for already-stranded entries: walks every UUID's
    `midhash256` field to build the orphan index, merges each stranded
    `file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
-   deletes the stranded root, and re-points the alias.
+   deletes the stranded root, and re-points the alias. *Removed in
+   2026-10* with the SQLite switch, together with the other one-shot Redis
+   migrations; it had been run on watch.nsl.sh before the migration.
 
 Verified on the dev stack: 48 stranded entries were reunited; the
 editor now shows every file with its full plugin metadata under one

docs/meta-core-architecture.md

@@ -3,14 +3,15 @@
 ## Document Purpose
 
 This document describes the architectural design of **meta-core**, the
-standalone container that owns Redis, the leader-info file, the HTTP+SSE
+standalone container that owns the metadata store (an embedded SQLite
+file; Redis until 2026-10), the leader-info file, the HTTP+SSE
 metadata surface, and a WebDAV file server. It covers the leader gate,
 storage data model, API design, the SSE event surface, and the integration
 contract with other MetaMesh services.
 
-For the on-disk Redis layout, see
+For the on-disk key layout, see
 [metadata-storage-structure.md](metadata-storage-structure.md). For the
-external-access design (Redis lockdown, SSE wire contract, auth rules),
+external-access design (store lockdown, SSE wire contract, auth rules),
 see [api-mediated-access.md](api-mediated-access.md). For the UUID-rooted
 storage rationale, see [uuid-rooted-metadata.md](uuid-rooted-metadata.md).
 
@@ -31,14 +32,15 @@ view of file metadata:
 
 meta-core sits below all of these as the single owner of:
 
-1. **Redis** — the metadata store and event source.
+1. **The metadata store** — an embedded SQLite file
+   (`/meta-core/db/meta-core.db`) holding metadata and the event streams.
 2. **The leader-info file** — the one place services look to discover meta-core's URLs.
 3. **The HTTP+SSE surface** — the only way external services touch the data plane.
 4. **A WebDAV server** — file access for plugins and other services.
 
 ### Core design principle
 
-> External services never speak Redis. Every read, write, and event flows
+> External services never touch the store. Every read, write, and event flows
 > through meta-core's HTTP+SSE surface.
 
 This is the contract that PR D of [api-mediated-access.md](api-mediated-access.md)
@@ -53,8 +55,8 @@ landed and that the rest of this document assumes.
 meta-core runs as a dedicated container (`metacore-app`). The dev compose
 runs a single instance; the legacy multi-instance topology (sibling
 followers) was deprecated. The flock primitive is retained as the
-correctness anchor — it guarantees that at most one process owns Redis and
-the writable state under `/meta-core`.
+correctness anchor — it guarantees that at most one process writes the
+database file and the writable state under `/meta-core`.
 
 ```
 ┌────────────────────────────────────────────────────────────────────────┐
@@ -62,17 +64,17 @@ the writable state under `/meta-core`.
 │                                                                        │
 │  supervisord                                                           │
 │   ├─ leader-election.sh   (flock → write kv-leader.info → exec self)   │
-│   ├─ redis-server         (AOF + RDB on /meta-core/db/redis)           │
 │   ├─ nginx                (TLS termination, proxy_cache for /webdav)   │
 │   ├─ rclone rcd           (mount daemon)                               │
 │   ├─ mount-watcher.sh     (rclone mount lifecycle)                     │
 │   └─ meta-core (Go) ──►   HTTP + SSE + WebDAV on :9000                 │
+│                           + embedded SQLite (db/meta-core.db)          │
 └────────────────────────────────────────────────────────────────────────┘
                                     │
        ┌────────────────────────────┴────────────────────────────┐
        │  /meta-core (shared volume)                             │
        │    locks/kv-leader.{lock,info}                          │
-       │    db/redis/                                            │
+       │    db/meta-core.db (+ -wal, -shm)                       │
        │    services/*.json                                      │
        │    mounts/{mounts.json, errors/}                        │
        │    watchers.json                                        │
@@ -152,7 +154,7 @@ Container startup
        ▼
 ┌──────────────────────────────┐
 │ supervisord starts:          │
-│   redis → nginx → rclone →   │
+│   nginx → rclone →           │
 │   meta-core Go binary        │
 └──────────────────────────────┘
 ```
@@ -167,10 +169,11 @@ gate wins. Therefore:
   local hostname + IP + config and returns it on demand. There are no
   `OnBecomeLeader` / `OnBecomeFollower` / `OnLeaderLost` callbacks; those
   states do not exist inside the Go process.
-- `LeaderLockInfo.RedisUrl` is intentionally empty in the response from
-  `/urls` — see `internal/leader/election.go:82`. PR D of
-  api-mediated-access.md removed direct Redis exposure; consumers route
-  metadata I/O through meta-core's HTTP API and SSE event streams.
+- `LeaderLockInfo.RedisUrl` is always empty in the response from
+  `/urls` (kept for wire compatibility). PR D of
+  api-mediated-access.md removed direct Redis exposure, and since 2026-10
+  there is no Redis at all; consumers route metadata I/O through
+  meta-core's HTTP API and SSE event streams.
 
 ### Lock-info file
 
@@ -192,18 +195,18 @@ Other services read this file, then call `GET /urls` for full discovery
   `metacore-app` is provisioned, its bash loop will acquire on next tick.
   The dev compose runs a single instance; failover is "restart the
   container."
-- **Redis crash inside the container** → supervisord restarts redis. The
-  Go binary's connect loop retries up to 30 times (1s delay) at startup;
-  at steady state, individual operations fail and clients retry.
+- **The store is in-process** — there is no separate database process to
+  crash. A meta-core crash is a process restart; SQLite's WAL recovers the
+  last committed transaction on reopen.
 
 ---
 
 ## Schema-Version Gate
 
 Before serving any traffic, `cmd/meta-core/main.go:54-61` calls
-`storage.EnsureSchemaVersion`. The sentinel lives in Redis at
+`storage.EnsureSchemaVersion`. The sentinel lives in the store at
 `meta-core:schema-version` (`internal/storage/schema_version.go`). The
-gate refuses to boot when the existing Redis layout predates the
+gate refuses to boot when the existing store layout predates the
 UUID-rooted schema, so legacy `file:midhash256:*` data can't leak into
 the current code paths. In alpha there is no automated migration —
 operators wipe with `pnpm run clean:all` and restart.
@@ -216,20 +219,20 @@ operators wipe with `pnpm run clean:all` and restart.
 
 ```
 ┌────────────────────────────────────────────────────────────────────┐
-│                       METADATA  (Redis)                            │
+│                 METADATA  (embedded SQLite, kv table)              │
 │                                                                    │
 │  Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars)     │
 │  Storage: flat string keys                                         │
 │           file:<uuid>/<property>           → string value          │
 │           file:<uuid>/cids/<bareCid>       → "true" (key-set)      │
-│           file:<uuid>/duplicates           → SET of paths          │
+│           dup:<uuid>/<path>                → "" (one per path)     │
 │                                                                    │
 │  No stored canonical_cid — derived by rank on read.                │
 │                                                                    │
-│  Aliases (reverse index, one Redis STRING each):                   │
+│  Aliases (reverse index, one key each):                            │
 │           cid:<bareCid>                    → <uuid>                │
 │                                                                    │
-│  Index:   file:__index__                   → SET of all UUIDs      │
+│  Index:   idx:roots/<uuid>                 → "" (one per root)     │
 │                                                                    │
 │  Mintable without a local file (remote / deleted entries OK).      │
 │  Resolvable by any registered CID in O(1).                         │
@@ -276,9 +279,8 @@ Bootstrap / discovery
   GET  /status
   GET  /leader
   GET  /urls
-  GET  /services         (alias: /api/services)
-  GET  /services/{name}
-  GET  /services/cleanup/stats
+  GET  /neighbors        (alias: /api/neighbors)
+  GET  /api/stats
 
 Metadata (primary surface for other services)
   GET    /meta
@@ -318,7 +320,7 @@ Editor / KV / schema / snapshot
   POST /api/snapshot/import
   POST /api/snapshot/wipe
 
-SSE event streams (HTTP-mirror of Redis Streams)
+SSE event streams (HTTP-mirror of the store's event streams)
   GET /api/events/files
   GET /api/events/meta
 
@@ -338,11 +340,10 @@ Watchers
   POST /api/watchers/{scan-all,reset-all}
   (deprecated: /api/scan/trigger, /api/scan/status — see watcher/handlers.go:32)
 
-Admin
-  POST /api/admin/migrate-dual-roots
-  POST /api/admin/sweep-literature-echoes   (dry-run unless ?apply=true;
-       strips query echoes off domain=literature fileType=card records —
-       see storage/literature_echo_sweep.go)
+(The one-shot Redis-era admin migrations — `/api/admin/migrate-dual-roots`,
+`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes` —
+were removed with the 2026-10 SQLite switch; they had been run once on
+watch.nsl.sh beforehand.)
 
 WebDAV
   /webdav/...   (GET, PUT, DELETE, MKCOL, COPY, MOVE, PROPFIND)
@@ -370,12 +371,16 @@ See `docs/api-mediated-access.md` "Auth" for the rationale.
 ### SSE event streams — wire contract
 
 `/api/events/files` and `/api/events/meta` are SSE mirrors of the
-`file:events` and `meta:events` Redis Streams. The contract
+`file:events` and `meta:events` streams, stored as `ev:<stream>/…` rows of
+the kv table (`internal/kv/stream.go`). The contract
 (`internal/api/sse_events.go`):
 
-- One SSE event per Redis Stream entry.
-- `id:` is the opaque `<ms>-<seq>` Redis Stream entry ID — clients echo
-  it back via `Last-Event-ID` on reconnect.
+- One SSE event per stream entry.
+- `id:` is the opaque `<ms>-<seq>` entry ID (the Redis Stream form, kept so
+  cursors persisted before the 2026-10 switch still compare) — clients echo
+  it back via `Last-Event-ID` on reconnect. Without `Last-Event-ID` the
+  stream starts at the head (resolved once at connect); `0` / `0-0` replay
+  from the beginning.
 - `event:` is the `type` field of the underlying entry.
 - `data:` is the JSON of the rest of the entry.
 - Heartbeats: SSE comment `:keep-alive\n\n` every 30s of silence.
@@ -384,9 +389,9 @@ See `docs/api-mediated-access.md` "Auth" for the rationale.
   `event: gap` payload before resuming from the oldest available entry.
 
 External services use SSE with `Last-Event-ID` as the resume primitive.
-A Redis consumer-group (XREADGROUP / XACK) exists *internally* — the
-`MetaPublisher` uses one to drain keyspace notifications — but it is not
-the external contract.
+`meta:events` entries are appended by the storage engine in the same
+transaction as the change they describe; the Redis-era keyspace-notification
+hop through `MetaPublisher` is gone.
 
 ### WebDAV URLs
 
@@ -416,7 +421,7 @@ WebDAV caching is handled by nginx's `proxy_cache` (see
 | Single binary | Static compilation |
 | Low resource | ~10-20 MB resident |
 | HTTP server | stdlib `net/http` + gorilla/mux |
-| Redis client | go-redis |
+| Metadata store | embedded SQLite via `modernc.org/sqlite` (pure Go, CGO off) |
 | WebDAV | `golang.org/x/net/webdav` |
 
 ### Repo layout
@@ -429,12 +434,13 @@ packages/meta-core/
 │   ├── cid/                      CID parsing + algorithm ranking
 │   ├── config/                   env-driven configuration
 │   ├── discovery/                service registry + dead-service cleaner
-│   ├── events/                   meta:events stream publisher
+│   ├── events/                   boot-time meta:events republish
+│   ├── kv/                       SQLite kv engine + event streams
 │   ├── leader/                   LeaderInfoProvider (no election logic)
 │   ├── mounts/                   rclone mount manager + handlers + stats
 │   ├── schema/                   live per-field schema indexer
 │   ├── snapshot/                 snapshot export / import / wipe
-│   ├── storage/                  Redis wrapper + UUID + CID reverse index + schema gate
+│   ├── storage/                  flat-key records + search + UUID + CID reverse index + schema gate
 │   ├── watcher/                  scanner + dispatcher + midhash + state
 │   ├── watchers/                 polling-based watcher configs
 │   └── webdav/                   /webdav/* handler
@@ -455,8 +461,8 @@ packages/meta-core/
 ```
 1. Load configuration from environment
 2. Build LeaderInfoProvider          (no election; bash gate already won)
-3. Connect to local Redis            (retry up to 30× / 1s)
-4. EnsureSchemaVersion               (abort if Redis layout is stale)
+3. Open the SQLite store             (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
+4. EnsureSchemaVersion               (abort if the store layout is stale)
 5. Start service discovery + dead-service cleaner
 6. NewServer + Server.Start          (boots watcher, mounts, WebDAV, API)
 7. On first connection: start MetaPublisher + schema Indexer,
@@ -467,9 +473,9 @@ packages/meta-core/
 
 ### Process management
 
-Redis is not spawned by the Go binary; supervisord starts it before
-meta-core (`docker/supervisord.conf`). The Go binary connects to
-`localhost:6379` and retries until Redis is reachable.
+There is no database process: the store is embedded in the Go binary
+(`docker/supervisord.conf` runs meta-core, nginx and rclone). Until
+2026-10 supervisord also ran `redis-server` on `localhost:6379`.
 
 ---
 
@@ -477,15 +483,16 @@ meta-core (`docker/supervisord.conf`). The Go binary connects to
 
 ### Container restart
 
-Supervisord restarts crashed processes (meta-core, redis, nginx, rclone)
+Supervisord restarts crashed processes (meta-core, nginx, rclone)
 individually. The bash flock is held by supervisord's process tree, so
 internal crashes don't trigger re-election.
 
-### Redis crash
+### Store durability
 
-Supervisord restarts redis. The Go binary's connect retry handles the
-window. Persistence is RDB + AOF on the `/meta-core` volume, so no data is
-lost across restarts.
+The SQLite file runs in WAL mode with `synchronous=NORMAL`: every
+committed transaction survives a meta-core process crash. A record write — fields, root index, CID aliases,
+FTS row and its events — is one transaction, so a crash never leaves a
+half-written record.
 
 ### Container crash
 
@@ -518,7 +525,10 @@ via `/health`; metadata I/O and SSE event streams stop.
 | `SERVICE_VERSION` | `1.0.0` | Reported in `/status`. |
 | `API_PORT` | `8180` | External port baked into `baseUrl`. |
 | `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). |
-| `REDIS_PORT` | `6379` | Local Redis port. |
+| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` (named volume — SQLite WAL is unsafe on the 9p `/meta-core` bind of Windows dev boxes). |
+| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection. |
+| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
+| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
 | `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
 | `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind address. |
 | `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
@@ -537,7 +547,7 @@ via `/health`; metadata I/O and SSE event streams stop.
 |---|---|
 | `locks/kv-leader.lock` | Flock target. |
 | `locks/kv-leader.info` | Leader API URL (plaintext). |
-| `db/redis/{dump.rdb, appendonly.aof}` | Redis persistence. |
+| `db/meta-core.db` (+ `-wal`, `-shm`) | Metadata store (SQLite). Replaced `db/redis/` in 2026-10. |
 | `services/*.json` | Service registry. |
 | `mounts/mounts.json` | rclone mount configuration. |
 | `mounts/errors/{id}.error` | Mount error logs. |
@@ -553,5 +563,5 @@ and file access for MetaMesh into one container with a typed HTTP+SSE
 surface. The bash flock gate guarantees single-leader semantics without
 an external consensus system. UUID-rooted storage with CID aliasing
 treats all content hashes symmetrically and makes any-CID resolution
-O(1). API-mediated access ensures no other service ever needs to speak
-Redis.
+a single lookup. API-mediated access ensures no other service ever touches
+the store.

docs/metadata-storage-structure.md

@@ -1,6 +1,6 @@
-# Redis Metadata Storage Structure
+# Metadata Storage Structure
 
-How meta-core lays out file metadata in Redis. This document covers the
+How meta-core lays out file metadata in its store. This document covers the
 on-disk shape (UUID-rooted flat string keys, CID reverse index,
 per-root sets) and the schema-version gate. It does **not** enumerate
 field semantics — that's the job of the repo-root
@@ -12,40 +12,80 @@ reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
 For the external-access contract (HTTP-only, SSE for events), see
 [`api-mediated-access.md`](api-mediated-access.md).
 
+> **Storage engine (since 2026-10): an embedded SQLite file, not Redis.**
+> Until 2026-10 these keys lived in a Redis server spawned inside the
+> container. They now live in one string→string table of an embedded SQLite
+> database (pure-Go driver `modernc.org/sqlite`, CGO off). The **key names
+> did not change** — only the Redis *types* that some bookkeeping used (SETs,
+> HASHes, Streams) were flattened into plain keys, listed below. Holding
+> everything in RAM had cost ~6.6 GB on watch.nsl.sh's 169k records (Redis
+> plus the API's in-memory search copy) and caused repeated OOM kills.
+
 ## Quick summary
 
+- **One table**: `kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`,
+  schema-free — a new metadata field is just a new key, never a migration.
+  File: `/meta-core/db/meta-core.db` (override `META_CORE_DB_PATH`), WAL
+  mode, `synchronous=NORMAL`.
 - **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
   not content hashes. Examples in this doc use the placeholder
   `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
-- **Properties are flat Redis STRING keys**, one per leaf:
-  `file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
+- **Properties are flat string keys**, one per leaf:
+  `file:<uuid>/<property/path>` → string value. A record is exactly the
+  key range `[file:<uuid>/, file:<uuid>0)` — there is no per-record field
+  list.
 - **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
-  (a Redis STRING per known CID; no algorithm prefix — the multicodec
+  (one key per known CID; no algorithm prefix — the multicodec
   already carries it).
-- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
-  `"true"` per known CID, midhash included), `file:<uuid>/duplicates`
-  (alternate paths with the same content), plus `file:__index__` (every UUID).
+- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one key
+  `"true"` per known CID, midhash included), `dup:<uuid>/<path>` (alternate
+  paths with the same content), plus `idx:roots/<uuid>` (every UUID).
   There is **no stored `canonical_cid`** — it is derived by rank on read.
 - **A schema-version sentinel** (`meta-core:schema-version`) refuses to
   boot against legacy data.
 
-Implementation lives in `internal/storage/client.go`,
-`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
-`internal/storage/schema_version.go`.
+Implementation lives in `internal/kv/kv.go` (engine),
+`internal/kv/stream.go` (event streams), `internal/storage/client.go`,
+`internal/storage/search.go`, `internal/storage/cid_resolution.go`,
+`internal/storage/uuid.go`, and `internal/storage/schema_version.go`.
 
 ## Key structure
 
+Everything is one string→string table. The full layout:
+
+| Key | Value | Notes |
+|---|---|---|
+| `file:<rid>/<field>` | field value | a record property — the **only** thing under `file:` |
+| `cid:<cid>` | `<rid>` | reverse index |
+| `meta-core:schema-version` | `"2"` | schema gate |
+| `idx:roots/<rid>` | `""` | set of every root (was the Redis SET `file:__index__`) |
+| `idx:fts/<rid>` | FTS rowid | the record's row in the FTS table |
+| `dup:<rid>/<path>` | `""` | a duplicate path (was the SET `file:<rid>/duplicates`) |
+| `udl:rec:<uid>/<cid>/<key>` | JSON `{record,version,ts,value?}` | User Data Layer entry (was a HASH) |
+| `udl:idx:…<set>\x1f<member>` | `""` | UDL index sets, one key per member |
+| `ev:<stream>/<ms13>-<seq10>` | JSON of the entry's fields | `meta:events` / `file:events` entries (were Redis Streams) |
+| `evhwm:<stream>` | last issued ID | stream high-water mark |
+| `btmagnet:*`, `searchcov:*`, `nzb:*` | — | written by other services via `/api/kv/value`; unchanged |
+
+Keeping bookkeeping **out of `file:`** is load-bearing: the `file:` range is
+exactly the records, so the field index covers only real fields and every
+change under `file:` is a real field change worth publishing on
+`meta:events`. The per-record `file:<rid>/__fields__` SET and the
+`file:__dirty__` set of the Redis era no longer exist. Roots are split from
+the field at the **first** `/`, so CID-rooted and UUID-rooted records both
+parse.
+
 ### Roots
 
 ```
-file:<uuid>/<property/path>   →   Redis STRING (always)
+file:<uuid>/<property/path>   →   string (always)
 ```
 
 UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
 UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
-time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
-in a given window, which restored some of the debug-ability lost when
-midhash256 stopped being the root.
+time-sortable prefix means roots minted in a given window sort together
+(`k >= 'file:01JKR8' AND k < 'file:01JKR9'`), which restored some of the
+debug-ability lost when midhash256 stopped being the root.
 
 Properties use `/` as the path separator. Nested objects in source code
 are flattened to flat keys on write and reconstructed on read. Examples:
@@ -62,7 +102,7 @@ file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster                       = "bafkreih...
 
 ### Reverse-index aliases (CIDs)
 
-Every CID known for a file is registered as its own Redis STRING, keyed by
+Every CID known for a file is registered as its own key, keyed by
 the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
 function in the multicodec, so naming it again in the key would be
 redundant (and would let the same digest be indexed under two spellings):
@@ -78,13 +118,12 @@ cid:bafkreih5kapbjzq...   →   01JKR8XW5T4QMV7AHNJ2DEFGHK
 cid:bagacbabaec7v3...     →   01JKR8XW5T4QMV7AHNJ2DEFGHK
 ```
 
-The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
+The reverse index makes `GET /api/meta/<cid>` a primary-key lookup for any
 registered CID, independent of which hash algorithm — there is no primary.
-The auto-alias hook (`maybeAddAliasFromFieldLocked` in
-`internal/storage/cid_resolution.go`, called from `SetProperty`,
-`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
-field whose name starts with `cids/`, so writers emit key-set members
-without knowing the reverse index exists.
+The record write path (`writeFieldsTx` in `internal/storage/client.go`, used
+by `SetProperty`, `SetMetadataFlat` and `MergeMetadataFlat`) registers the
+alias for any field whose name starts with `cids/`, in the same transaction,
+so writers emit key-set members without knowing the reverse index exists.
 
 > ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
 > `midhash256` *named* field written to a root is stored verbatim and is
@@ -95,12 +134,13 @@ without knowing the reverse index exists.
 ### Per-root sets
 
 ```
-file:<uuid>/cids/<bareCid>    →   STRING "true" — one key per known CID (a key-set, not a Redis SET)
-file:<uuid>/duplicates        →   Redis SET of alternate paths with same content
+file:<uuid>/cids/<bareCid>    →   "true" — one key per known CID (a key-set)
+dup:<uuid>/<path>             →   ""     — one key per alternate path with same content
 ```
 
-`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
-which keeps it conflict-free to merge across peers (a SET would not be).
+`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` key per member,
+which keeps it conflict-free to merge across peers (a single set-valued
+field would not be).
 The record's own address (the midhash) is just the `0x1000` member — it has
 no separate field.
 
@@ -114,61 +154,88 @@ was removed with the flat `cid_*` keys.
 ### Global index
 
 ```
-file:__index__   →   Redis SET of all UUIDs
+idx:roots/<uuid>   →   ""   (one key per root; was the Redis SET file:__index__)
 ```
 
-`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
-every file without scanning the keyspace.
+`internal/storage/client.go::GetAllHashIDs` range-reads `idx:roots/` to
+enumerate every file without scanning the records.
 
-```bash
-SMEMBERS file:__index__         # all UUIDs
-SCARD    file:__index__         # total file count
-SISMEMBER file:__index__ <uuid>
-```
+### Search structures (derived)
+
+Two derived structures are maintained on top of `kv`, in the same
+transaction as every record write. Both are rebuildable from the kv rows and
+never authoritative (`internal/storage/search.go`):
+
+- **`kv_field`** — a partial expression index
+  `ON kv(lower(substr(k,instr(k,'/')+1)), lower(v)) WHERE k>='file:' AND k<'file;' AND instr(k,'/')>0`.
+  It answers "which records have field F with a value like V" for **any**
+  field, including ones invented tomorrow — the index is over the key itself.
+- **`fts`** — an FTS5 trigram table `fts(rid UNINDEXED, hay)` holding one row
+  per record: its search haystack plus its rid (`idx:fts/<rid>` points at the
+  row). Trigrams make every 3+ character substring indexable.
+
+`POST /api/metadata/search` uses SQL to build a superset of candidates, then
+verifies each with the unchanged Go matchers (`recordMatchesFilter`,
+`matchAllTokens`), so semantics are identical to the old in-memory index.
+Results still come in hashID order and are fresh immediately (no refresh
+lag). This replaced the API's in-memory `searchIndex` (60 s refresh, ~30 KB
+RAM per record).
 
 ### Schema-version sentinel
 
 ```
-meta-core:schema-version   →   STRING (currently "2")
+meta-core:schema-version   →   string (currently "2")
 ```
 
 Set at first boot, checked on every subsequent boot
 (`internal/storage/schema_version.go`). If the sentinel is missing but
-`file:__index__` is non-empty, the gate refuses to boot (legacy data).
+the root index is non-empty, the gate refuses to boot (legacy data).
 If the sentinel is present but doesn't match `SchemaVersion`, the gate
 also refuses to boot. Alpha-stage policy: operators wipe and restart;
 there is no automated migration.
 
-## Why flat STRING keys?
+## Why flat string keys?
 
-The store does not use Redis Hashes. The reasons are:
+The store keeps one flat key per leaf rather than a document or a
+per-record row. The reasons are:
 
-1. **Field-level keyspace notifications.** Redis publishes
-   `__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
-   you'd only get one notification on the hash itself. meta-core's
-   `MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
-   notifications and publishes structured events to the `meta:events`
-   Redis Stream.
-2. **Selective updates without read-modify-write.** A plugin writing
+1. **Schema-free.** A plugin inventing a field writes a new key; nothing
+   declares it, and the table never needs a migration. The field index
+   covers it automatically.
+2. **Conflict-free merging across peers.** Key-sets (`cids/<cid>`, and the
+   flat fields in general) merge by union of keys — no read-modify-write of
+   a shared value, nothing to reconcile.
+3. **Selective updates without read-modify-write.** A plugin writing
    `tmdb/poster` doesn't have to read the rest of the document.
-3. **Compatibility with the CID alias hooks.** Field writes can trigger
-   alias-maintenance work in `SetProperty` without the plugin caring.
+4. **Compatibility with the CID alias hooks.** Field writes can trigger
+   alias-maintenance work in the write path without the plugin caring.
+
+(Under Redis there was a fourth reason — field-level keyspace notifications
+fed `meta:events`. That is obsolete: events are now written by the storage
+engine itself, see [Event streams](#event-streams).)
 
 ## Reads and writes
 
+All writes go through **one writer goroutine** with group commit: concurrent
+writes share one SQLite transaction, each under its own SAVEPOINT so a
+failure rolls back alone. A record write — fields + root index + CID aliases
++ FTS row + events — is atomic. (Measured ~835 records/s with 16 concurrent
+writers of 40-field records.) Reads use a separate connection pool and see
+the last committed state.
+
 The Go API in `internal/storage/client.go`:
 
 | Method | Behaviour |
 |---|---|
-| `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
-| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
+| `GetMetadataFlat(uuid)` | Range-read `[file:<uuid>/, file:<uuid>0)`, return `map[string]string`. |
+| `SetMetadataFlat(uuid, m)` | Put each field; add `idx:roots/<uuid>`; auto-register an alias for every `cids/<cid>` member; refresh the FTS row. One transaction. |
 | `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
-| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
-| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
-| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
+| `GetProperty(uuid, field)` | Get `file:<uuid>/<field>`. |
+| `SetProperty(uuid, field, value)` | Put `file:<uuid>/<field>`; auto-register alias if the field is a CID. |
+| `DeleteRoot(uuid)` | Delete the `file:<uuid>/` range, its duplicates, search row and index membership; unmap every reverse-index entry. |
 | `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
 | `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
-| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
+| `ResolveRoot("<bareCid>")` | Get `cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
 | `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
 
 ## Reconstruction (flat → nested)
@@ -196,7 +263,7 @@ Rules:
 
 ## Plugin contribution flow
 
-Plugins don't talk to Redis. They write metadata via meta-core's HTTP
+Plugins don't talk to the store directly. They write metadata via meta-core's HTTP
 API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
 writes under a prefix:
 
@@ -215,12 +282,24 @@ deliberately does not duplicate that registry — it would drift.
 
 ## Event streams
 
-Two Redis Streams are produced inside meta-core:
+Two event streams are kept inside the store, as rows
+`ev:<stream>/<ms13>-<seq10>` (zero-padded so key order is ID order) plus a
+high-water mark `evhwm:<stream>` (`internal/kv/stream.go`):
 
 | Stream | Producer | Contents |
 |---|---|---|
 | `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
-| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
+| `meta:events` | the storage engine (`internal/kv`) | `{type: set\|del, key, ts}` for every Put/Delete of a `file:` key, appended **in the same transaction** as the change. |
+
+`del` is emitted only when the key existed; `set` is emitted even for an
+unchanged value (Redis semantics, kept). Retention is ~100k entries,
+trimmed every 10k appends. At boot `MetaPublisher`
+(`internal/events/meta_publisher.go`) still republishes every field into
+`meta:events` (meta-fuse rebuilds its VFS only by replaying from `0-0`),
+batched 5k events per transaction.
+
+The Redis-era path — keyspace notifications drained by `MetaPublisher`
+through a PSUBSCRIBE / consumer-group hop — is gone (2026-10).
 
 External consumers do **not** read these streams directly. They subscribe
 via SSE:
@@ -228,51 +307,50 @@ via SSE:
 - `GET /api/events/files` — mirror of `file:events`
 - `GET /api/events/meta` — mirror of `meta:events`
 
-The SSE wire contract uses `Last-Event-ID` for resume (the opaque
-`<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
+The SSE wire contract uses `Last-Event-ID` for resume. IDs keep the Redis
+`<ms>-<seq>` form so cursors persisted by meta-sort / meta-share /
+meta-stremio across the switch still compare. Heartbeats are SSE comments
 (`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
 a client's cursor, the handler emits one `event: gap` payload before
 resuming. See `internal/api/sse_events.go` and
 [`api-mediated-access.md`](api-mediated-access.md) for the full
 contract.
 
-Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
-inside `MetaPublisher` to drain keyspace notifications, but this is not
-the external surface — external services use SSE.
-
 ## Debugging
 
-```bash
-# Roots
-docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
-docker exec metacore-app redis-cli SCARD    'file:__index__'
-docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
+The database path is `/meta-core/db/meta-core.db` in the image default and
+on the store app; dev stacks use `/meta-core-db/meta-core.db` (a named
+volume — the `/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL
+is unsafe). Always open it **read-only**; writes must go through the API.
 
-# Full document for one root (flat scan)
-docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
+```bash
+DB=/meta-core/db/meta-core.db      # dev stack: /meta-core-db/meta-core.db
+Q() { docker exec metacore-app sqlite3 -readonly "$DB" "$1"; }
+
+# Roots
+Q "SELECT count(*) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0'"
+Q "SELECT substr(k,11) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0' LIMIT 10"
+
+# Full document for one root
+Q "SELECT k,v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'"
 
 # One property
-docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
-
-# Confirm keys are STRING (not HASH)
-docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
-# string
+Q "SELECT v FROM kv WHERE k = 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'"
 
 # Reverse-index lookup (bare CID — no algorithm prefix)
-docker exec metacore-app redis-cli GET 'cid:bafkreih...'      # → uuid
+Q "SELECT v FROM kv WHERE k = 'cid:bafkreih...'"                   # → uuid
 
-# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
-docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
+# The cids key-set
+Q "SELECT k FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids0'"
 
 # canonical_cid is NOT stored. Derive it via the API instead:
 curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
 
 # Schema-version sentinel
-docker exec metacore-app redis-cli GET 'meta-core:schema-version'
+Q "SELECT v FROM kv WHERE k = 'meta-core:schema-version'"
 
 # Streams (server-side; external consumers should use SSE)
-docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
-docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
+Q "SELECT k,v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10"
 
 # HTTP equivalents — preferred
 curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>'      # CID → document
@@ -286,12 +364,14 @@ curl -k 'https://metacore-dev.localhost:8083/api/kv/info'         # KV stats
 
 | Component | Path |
 |---|---|
-| Redis client + flat-key ops | `internal/storage/client.go` |
+| Storage engine (SQLite kv table, writer, group commit) | `internal/kv/kv.go` |
+| Event streams as kv rows | `internal/kv/stream.go` |
+| Flat-key record ops | `internal/storage/client.go` |
+| Search (field index + FTS) | `internal/storage/search.go` |
 | UUIDv7 minting | `internal/storage/uuid.go` |
-| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
+| CID reverse index + aliases + canonical-CID derivation | `internal/storage/cid_resolution.go` |
 | Schema-version gate | `internal/storage/schema_version.go` |
-| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
-| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
+| Boot-time `meta:events` republish | `internal/events/meta_publisher.go` |
 | Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
 | SSE event mirror | `internal/api/sse_events.go` |
 | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |

docs/uuid-rooted-metadata.md

@@ -20,6 +20,12 @@
 >    `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
 >    function, so the `<algo>:` prefix was redundant.
 >
+> 3. **The store is no longer Redis (2026-10).** The same keys now live in
+>    one string→string table of an embedded SQLite file; Redis SETs became
+>    one key per member (`file:__index__` → `idx:roots/<uuid>`,
+>    `file:<uuid>/duplicates` → `dup:<uuid>/<path>`). Redis types named
+>    below are the shape at the time.
+>
 > Text below that references either shape describes the *proposal*, not the
 > code. The rank ordering and everything else did ship as written.
 
@@ -135,7 +141,7 @@ sole writer is meta-core itself:
 | Key | Type (as shipped) | Purpose |
 |-----|------|---------|
 | `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
-| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
+| `duplicates` | Redis SET at the time; since 2026-10 one `dup:<uuid>/<path>` key per path | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
 | ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
 
 Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
@@ -148,7 +154,8 @@ for using sets where they're the right structure.~~ **Reversed for `cids`.**
 The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
 so that two peers merging the same record can never disagree about set
 membership. `duplicates` remained a Redis SET (it is leader-local and never
-merged across peers).
+merged across peers) until the 2026-10 SQLite switch, which flattened it to
+`dup:<uuid>/<path>` keys.
 
 ### Canonical CID selection
 

editor/src/api/snapshotApi.ts

@@ -81,7 +81,7 @@ export class SnapshotAPI {
   }
 
   /**
-   * Wipe Redis metadata. Pass scope=metadata.
+   * Wipe all stored metadata. Pass scope=metadata.
    */
   static async wipe(scope: 'metadata'): Promise<WipeResult> {
     const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {

editor/src/components/SnapshotPanel.tsx

@@ -43,7 +43,7 @@ export function SnapshotPanel() {
 
   const runWipe = async () => {
     const ok = window.confirm(
-      'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
+      'This deletes ALL stored metadata. Files on disk are not touched, but every manual ' +
         'edit you made via this editor will be lost. Continue?'
     );
     if (!ok) return;

go.mod

@@ -1,21 +1,25 @@
 module github.com/metazla/meta-core
 
-go 1.21
+go 1.26.0
 
 require (
-	github.com/alicebob/miniredis/v2 v2.31.1
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.1
 	github.com/mr-tron/base58 v1.3.0
-	github.com/redis/go-redis/v9 v9.4.0
 	golang.org/x/crypto v0.31.0
 	golang.org/x/net v0.21.0
+	modernc.org/sqlite v1.60.1
 )
 
 require (
-	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/yuin/gopher-lua v1.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/mattn/go-isatty v0.0.24 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	modernc.org/libc v1.77.1 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.12.1 // indirect
 )
+
+require golang.org/x/sys v0.48.0

go.sum

@@ -1,36 +1,62 @@
-github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
-github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
-github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
-github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
-github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
-github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
-github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
 github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
-github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
+github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
+github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
 github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
 github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
-github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
-github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
-github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
+golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
+modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
+modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
+modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
+modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
+modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
+modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
+modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
+modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
+modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
+modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
+modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
+modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
+modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
+modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
+modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
+modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
+modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

internal/api/admin_migrate.go

@@ -1,101 +0,0 @@
-package api
-
-import (
-	"net/http"
-	"strconv"
-)
-
-// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
-// stranded midhash-rooted entries into their UUID roots. See
-// storage.MigrateDualRoots for the data-flow detail and
-// docs/api-mediated-access.md for the bug background.
-//
-// Returns {fixed: N} on success. Safe to call repeatedly — entries already
-// migrated are skipped on the next pass.
-func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	fixed, err := s.storage.MigrateDualRootsWithTimeout()
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"fixed":   fixed,
-	})
-}
-
-// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the
-// retired `domain=film|tv` values to `screen` and backfills `workForm` from
-// `contentKind` (METADATA_KEYS.md §14.17). See
-// storage.MigrateDomainScreen for why the vocabulary change is paid in the
-// data rather than with a read-side alias.
-//
-// Returns {fixed: N} — records changed, not records examined. Safe to call
-// repeatedly: a record already on the current vocabulary is skipped.
-func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	fixed, err := s.storage.MigrateDomainScreenWithTimeout()
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"fixed":   fixed,
-	})
-}
-
-// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
-// `anilistid` on a site card, and `workcid` / position keys on any card
-// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
-// why chapters and anchor cards' ids are deliberately left alone.
-//
-// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
-// keys, so a bare call only reports what it would delete. An unparseable
-// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
-// sweep that ran.
-//
-// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
-// records, byKey counts fields per key. Safe to call repeatedly: a swept
-// record plans nothing on the next pass.
-func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
-	if !s.storage.IsConnected() {
-		writeError(w, http.StatusServiceUnavailable, "storage not connected")
-		return
-	}
-
-	apply := false
-	if v := r.URL.Query().Get("apply"); v != "" {
-		parsed, err := strconv.ParseBool(v)
-		if err != nil {
-			writeError(w, http.StatusBadRequest, "apply must be true or false")
-			return
-		}
-		apply = parsed
-	}
-
-	res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
-	writeJSON(w, http.StatusOK, map[string]interface{}{
-		"success": true,
-		"dryRun":  !apply,
-		"planned": res.Planned,
-		"fixed":   res.Fixed,
-		"byKey":   res.ByKey,
-	})
-}

internal/api/admin_sweep_test.go

@@ -1,93 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/alicebob/miniredis/v2"
-
-	"github.com/metazla/meta-core/internal/storage"
-)
-
-// newSweepTestServer is a Server over miniredis with one literature card
-// carrying a copied `workcid`.
-func newSweepTestServer(t *testing.T) *Server {
-	t.Helper()
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
-	for k, v := range map[string]string{
-		"domain":   "literature",
-		"fileType": "card",
-		"workcid":  "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
-	} {
-		if err := c.SetProperty("card-1", k, v); err != nil {
-			t.Fatalf("seed %s: %v", k, err)
-		}
-	}
-	return &Server{storage: c}
-}
-
-type sweepResponse struct {
-	Success bool           `json:"success"`
-	DryRun  bool           `json:"dryRun"`
-	Planned int            `json:"planned"`
-	Fixed   int            `json:"fixed"`
-	ByKey   map[string]int `json:"byKey"`
-}
-
-func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
-	t.Helper()
-	rr := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
-	s.handleSweepLiteratureEchoes(rr, req)
-	var body sweepResponse
-	if rr.Code == http.StatusOK {
-		if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
-			t.Fatalf("decode %s: %v", rr.Body.String(), err)
-		}
-	}
-	return rr, body
-}
-
-// A bare call must never delete: the default is the dry run.
-func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
-	s := newSweepTestServer(t)
-
-	rr, body := postSweep(t, s, "")
-	if rr.Code != http.StatusOK {
-		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
-	}
-	if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
-		t.Errorf("dry run = %+v", body)
-	}
-	if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
-		t.Errorf("dry run deleted workcid")
-	}
-
-	rr, body = postSweep(t, s, "?apply=true")
-	if rr.Code != http.StatusOK {
-		t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
-	}
-	if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
-		t.Errorf("apply = %+v", body)
-	}
-	if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
-		t.Errorf("apply left workcid = %q", v)
-	}
-}
-
-// A typo must not pass for a sweep that ran.
-func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
-	s := newSweepTestServer(t)
-
-	rr, _ := postSweep(t, s, "?apply=yes-please")
-	if rr.Code != http.StatusBadRequest {
-		t.Errorf("status = %d, want 400", rr.Code)
-	}
-}

internal/api/files_register_test.go

@@ -9,24 +9,16 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
-// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
-// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
+// newRegisterTestServer wires handleRegisterFile to a temp store and a temp
+// FILES_PATH. Built directly, not through NewServer, so no watcher runs —
 // which is exactly the configuration this route exists for.
-func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
+func newRegisterTestServer(t *testing.T) (*Server, string) {
 	t.Helper()
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
 	files := t.TempDir()
-	return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
+	return &Server{storage: newTestStorage(t), config: &config.Config{FilesPath: files}}, files
 }
 
 func writeFixture(t *testing.T, files, rel, body string) string {
@@ -52,20 +44,20 @@ func postRegister(t *testing.T, s *Server, path string) (int, registerFileRespon
 	return rr.Code, resp
 }
 
-func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
+func streamLen(t *testing.T, s *Server) int {
 	t.Helper()
-	entries, err := mr.Stream("file:events")
+	n, err := s.storage.GetStreamLength("file:events")
 	if err != nil {
-		return 0
+		t.Fatalf("stream length: %v", err)
 	}
-	return len(entries)
+	return int(n)
 }
 
 // The whole point of the route: with the watcher off, a file written under
 // plugin/ becomes a record resolvable by its midhash, and meta-share hears
 // about it on file:events.
 func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
-	s, files, mr := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	rel := "plugin/subtitles/ab12cd34.eng.srt"
 	full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
 
@@ -85,20 +77,19 @@ func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
 		t.Fatalf("filePath = %q, want %q", fp, full)
 	}
 
-	entries, err := mr.Stream("file:events")
+	entries, err := s.storage.XRevRange("file:events", "+", "-", 0)
 	if err != nil || len(entries) != 1 {
 		t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
 	}
-	vals := strings.Join(entries[0].Values, "|")
-	if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
-		t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
+	if vals := entries[0].Values; vals["midhash256"] != resp.CID || vals["path"] != rel {
+		t.Fatalf("event must carry the midhash and the relative path, got %v", vals)
 	}
 }
 
 // Re-uploading the same subtitle is a no-op: same root, no second event (which
 // would make meta-share re-chunk and re-provide the file).
 func TestRegisterFile_IsIdempotent(t *testing.T) {
-	s, files, mr := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	rel := "plugin/subtitles/same.eng.srt"
 	writeFixture(t, files, rel, "same bytes")
 
@@ -110,7 +101,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
 	if second.Root != first.Root || second.CID != first.CID {
 		t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
 	}
-	if n := streamLen(t, mr); n != 1 {
+	if n := streamLen(t, s); n != 1 {
 		t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
 	}
 }
@@ -118,7 +109,7 @@ func TestRegisterFile_IsIdempotent(t *testing.T) {
 // Only plugin output may be registered here. /files/watch is the user's
 // library and belongs to the watcher; traversal must not escape either.
 func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
-	s, files, _ := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	writeFixture(t, files, "watch/movie.mkv", "x")
 	for _, p := range []string{
 		"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
@@ -131,7 +122,7 @@ func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
 }
 
 func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
-	s, files, _ := newRegisterTestServer(t)
+	s, files := newRegisterTestServer(t)
 	if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
 		t.Errorf("missing file: want 404, got %d", code)
 	}

internal/api/files_tuples_test.go

@@ -7,18 +7,12 @@ import (
 	"sort"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/metazla/meta-core/internal/storage"
 )
 
 func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
 	t.Helper()
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
+	c := newTestStorage(t)
 	return &Server{storage: c}, c
 }
 
@@ -105,7 +99,7 @@ func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
 }
 
 func TestFileTuples_StorageNotConnected(t *testing.T) {
-	s := &Server{storage: storage.NewClient("")}
+	s := &Server{storage: storage.NewClient()}
 
 	code, _ := getFileTuples(t, s, "")
 	if code != http.StatusServiceUnavailable {

internal/api/handlers.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 )
 
 // contentTypeByExt maps file extensions to MIME types
@@ -547,62 +548,49 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
 	http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
 }
 
-// handleListServices handles GET /services and GET /api/services
-// Supports optional ?current=<service-name> query parameter
-func (s *Server) handleListServices(w http.ResponseWriter, r *http.Request) {
-	services, err := s.discovery.DiscoverAll()
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
-	}
-
+// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
+//
+// Beacon v2 (docs/project-architecture/beacon-v2.md). The list is this node's
+// in-memory view of the UDP group, so it works across stacks that share a
+// network but no volume. Instances of one name collapse to the most recent
+// (one row per service, the nav menu's shape); `?all=1` returns every
+// instance, `?cap=<pattern>` keeps only nodes with a matching capability.
+func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
 	response := map[string]interface{}{
-		"services": services,
-		"count":    len(services),
-	}
-
-	// Add 'current' field if query parameter is provided
-	current := r.URL.Query().Get("current")
-	if current != "" {
-		response["current"] = current
+		"current":   s.config.ServiceName,
+		"neighbors": []interface{}{},
+		"count":     0,
 	}
 
+	if s.mesh == nil {
+		// Discovery off or failed to bind. Report empty rather than 503 —
+		// the nav is decoration and callers already treat [] as "hide".
+		response["enabled"] = false
 		writeJSON(w, http.StatusOK, response)
-}
-
-// handleGetService handles GET /services/{name}
-func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
-	vars := mux.Vars(r)
-	name := vars["name"]
-
-	if name == "" {
-		writeError(w, http.StatusBadRequest, "service name is required")
 		return
 	}
 
-	service, err := s.discovery.Discover(name)
-	if err != nil {
-		writeError(w, http.StatusInternalServerError, err.Error())
-		return
+	var list []meshdisco.Neighbor
+	if r.URL.Query().Get("all") != "" {
+		list = s.mesh.Neighbors()
+	} else {
+		list = s.mesh.NeighborsByName()
+	}
+	if pattern := r.URL.Query().Get("cap"); pattern != "" {
+		kept := list[:0]
+		for _, nb := range list {
+			if nb.Matches(pattern) {
+				kept = append(kept, nb)
+			}
+		}
+		list = kept
 	}
 
-	if service == nil {
-		writeError(w, http.StatusNotFound, "service not found")
-		return
-	}
-
-	writeJSON(w, http.StatusOK, service)
-}
-
-// handleCleanupStats handles GET /services/cleanup/stats
-func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
-	if s.cleaner == nil {
-		writeError(w, http.StatusServiceUnavailable, "cleaner not initialized")
-		return
-	}
-
-	stats := s.cleaner.Stats()
-	writeJSON(w, http.StatusOK, stats)
+	response["enabled"] = true
+	response["neighbors"] = list
+	response["count"] = len(list)
+	response["self"] = s.mesh.Self()
+	writeJSON(w, http.StatusOK, response)
 }
 
 // writeJSON writes a JSON response

internal/api/identity_gate_test.go

@@ -1,6 +1,6 @@
 package api
 
-// End-to-end exercise of the account gate against a live router + miniredis:
+// End-to-end exercise of the account gate against a live router + a temp store:
 // generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
 // both the key and every row are gone. Deliberately drives the real HTTP
 // handlers rather than the packages beneath them, because the bugs that matter
@@ -16,21 +16,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
 	"github.com/metazla/meta-core/internal/identity"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
 func newTestServer(t *testing.T) *Server {
 	t.Helper()
-	mr := miniredis.RunT(t)
-	stor := storage.NewClient("")
-	if err := stor.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect: %v", err)
-	}
-	t.Cleanup(func() { _ = stor.Close() })
+	stor := newTestStorage(t)
 
 	s := &Server{
 		config:     &config.Config{MetaCorePath: t.TempDir()},

internal/api/metadata.go

@@ -40,9 +40,9 @@ type MetadataSearchRequest struct {
 	Limit         int    `json:"limit,omitempty"`
 
 	// Filters is the structured form: AND across entries, OR within an
-	// entry's Values. Evaluated in memory against the search index alongside
-	// the free-text Query, so an id-anchored lookup costs a map probe per
-	// record rather than a Redis round-trip per record.
+	// entry's Values. Narrowed in SQL through the field index alongside the
+	// free-text Query, then verified per record (see serveIndexedSearch), so an
+	// id-anchored lookup costs an index probe rather than a read per record.
 	//
 	// Property/PropertyValue is the single-filter legacy shape and folds into
 	// this list (see normalizedFilters) with Exact=false, preserving its
@@ -445,9 +445,8 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Everything else — free text, structured filters, both, or neither
-	// (which returns the first `limit`) — is served from the in-memory search
-	// index (search_index.go). Matching and the returned metadata both come
-	// from memory, so a query costs zero Redis round-trips.
+	// (which returns the first `limit`) — is served from the store's derived
+	// search indexes (search_index.go → storage.Search).
 	//
 	// There is deliberately no second path. The property branch used to fall
 	// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which

internal/api/schema_handlers.go

@@ -3,10 +3,10 @@ package api
 import (
 	"encoding/json"
 	"net/http"
-	"time"
 )
 
-// handleSchemaGet returns the live per-field schema derived from meta:events.
+// handleSchemaGet returns the per-field schema (recomputed from the store every
+// few minutes; see schema.Indexer).
 func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
 	if s.schemaIndexer == nil {
 		http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
@@ -17,29 +17,26 @@ func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
 	_ = json.NewEncoder(w).Encode(snap)
 }
 
-// handleSchemaRescan re-emits a set event for every existing property via the
-// meta publisher, waits for the indexer to drain, and returns the refreshed
-// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
+// handleSchemaRescan recomputes the schema from a full scan of the store and
+// returns it. It no longer republishes every property into meta:events — the
+// indexer reads the store itself — so a rescan floods no consumer.
 func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
 	if s.schemaIndexer == nil {
 		http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
 		return
 	}
 
-	count, err := s.RepublishMetadata()
+	snap, err := s.schemaIndexer.Rebuild(r.Context())
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-
-	s.schemaIndexer.WaitForDrain(10 * time.Second)
-
-	snap := s.schemaIndexer.Snapshot()
-	snap.Source = "rescan"
+	out := *snap
+	out.Source = "rescan"
 
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(map[string]any{
-		"published_events": count,
-		"schema":           snap,
+		"published_events": 0,
+		"schema":           &out,
 	})
 }

internal/api/search_index.go

@@ -1,221 +1,59 @@
 package api
 
 import (
-	"log"
 	"net/http"
-	"sort"
 	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
 
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
-// from Redis in the background. Records ingest continuously, so a newly stored
-// record becomes searchable within one interval; the rebuild is a single
-// batched bulk read (storage.GetSearchIndexTuples), off the request path.
-const searchIndexRefreshInterval = 60 * time.Second
-
-// searchIndexReconcileEvery is how many refresh ticks pass between full
-// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full
-// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all.
-const searchIndexReconcileEvery = 30
-
-// searchIndex is an in-memory snapshot of every record's search haystack
-// (hashID + lowercased searchable fields), kept current from Redis. It exists
-// so POST /api/metadata/search matches in memory (string ops over a slice)
-// instead of doing a SCAN+MGET Redis round-trip per record — the latter took
-// 40s+ over ~72k records and made a live parallel search path unusable.
+// serveIndexedSearch answers a search from the database's derived indexes
+// (storage.Search): SQL narrows to a superset of candidates through the field
+// expression index and the FTS5 trigram table, then every candidate is checked
+// here with the exact matchers below — a token-AND match against the record's
+// haystack (or a whole-query hashID substring, preserving the legacy
+// behaviour) AND every structured filter.
 //
-// # Two update paths, and why
-//
-// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a
-// complete fresh copy. That is O(corpus) regardless of how much changed, and it
-// used to run every 60 s forever. Measured on a 19 622-record staging corpus
-// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side
-// held 783 MB for data Redis stores in 135 MB — largely the double copy.
-//
-// `refreshIncremental` re-reads only the records that changed, tracked by
-// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady
-// state; the full rebuild runs once at startup (there is nothing to be
-// incremental against) and then only as an explicit repair.
-//
-// The distinction matters because meta-core is the platform's permanent,
-// additive store — every search hit on every gateway is retained — so a rebuild
-// priced against the whole corpus is a cost that only ever grows.
-type searchIndex struct {
-	mu sync.RWMutex
-	// byID is the authoritative content; tuples is a stable-order view of it
-	// rebuilt on each update. Map iteration order in Go is randomised, so
-	// serving a limited query straight off the map would return a different
-	// subset each call — the slice is what keeps results stable.
-	byID   map[string]storage.SearchTuple
-	tuples []storage.SearchTuple
-	ready  atomic.Bool
-}
-
-// snapshot returns the current tuples slice and whether the index has built at
-// least once. Every update swaps in a fresh slice under the write lock (never
-// mutates in place), so the returned slice is safe to read lock-free.
-func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
-	idx.mu.RLock()
-	defer idx.mu.RUnlock()
-	return idx.tuples, idx.ready.Load()
-}
-
-// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds
-// the write lock.
-func (idx *searchIndex) rebuildViewLocked() {
-	view := make([]storage.SearchTuple, 0, len(idx.byID))
-	for _, t := range idx.byID {
-		view = append(view, t)
-	}
-	// Stable across rebuilds so a limited query returns a consistent page
-	// rather than a fresh random subset every interval.
-	sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID })
-	idx.tuples = view
-}
-
-// refresh rebuilds the whole index from Redis (a batched bulk read). Used for
-// the initial build and as an explicit repair — not the steady state.
-func (idx *searchIndex) refresh(stor *storage.Client) {
-	if stor == nil || !stor.IsConnected() {
-		return
-	}
-	tuples, err := stor.GetSearchIndexTuples()
-	if err != nil {
-		log.Printf("[API] search index refresh failed: %v", err)
-		return
-	}
-	byID := make(map[string]storage.SearchTuple, len(tuples))
-	for _, t := range tuples {
-		byID[t.HashID] = t
-	}
-	idx.mu.Lock()
-	idx.byID = byID
-	idx.rebuildViewLocked()
-	idx.mu.Unlock()
-	idx.ready.Store(true)
-}
-
-// refreshIncremental drains the dirty set and patches only those records.
-// Returns whether more dirty entries remain, so a burst is caught up over
-// consecutive passes rather than one per interval.
-//
-// A record in the dirty set but absent from storage was deleted and is evicted
-// here — otherwise it would keep answering searches until the next full
-// rebuild. A batch that fails after the (destructive) drain is re-marked by
-// storage so it is retried rather than silently lost.
-func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) {
-	if stor == nil || !stor.IsConnected() {
-		return false
-	}
-	ids, more, err := stor.DrainDirty()
-	if err != nil {
-		log.Printf("[API] search index drain failed: %v", err)
-		return false
-	}
-	if len(ids) == 0 {
-		return false
-	}
-	tuples, gone, err := stor.GetSearchIndexTuplesFor(ids)
-	if err != nil {
-		stor.MarkDirty(ids...)
-		log.Printf("[API] incremental search index refresh failed: %v", err)
-		return false
-	}
-
-	idx.mu.Lock()
-	if idx.byID == nil {
-		idx.byID = make(map[string]storage.SearchTuple, len(tuples))
-	}
-	for _, t := range tuples {
-		idx.byID[t.HashID] = t
-	}
-	for _, id := range gone {
-		delete(idx.byID, id)
-	}
-	idx.rebuildViewLocked()
-	idx.mu.Unlock()
-
-	log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples))
-	return more
-}
-
-// runRefreshLoop builds the index once in full, then keeps it current from the
-// dirty set, with a periodic full reconcile as a safety net.
-//
-// The reconcile exists because a *missed* dirty mark is silent: no error, just
-// a record whose indexed copy quietly diverges from storage until something
-// forces a rebuild. Marking is spread across eight write paths plus anything
-// added later, and the cost of getting it wrong is invisible — so the index
-// should not depend on that coverage being perfect forever. Every
-// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs
-// whatever drifted.
-//
-// That still drops the full rebuild from every 60 s to every 30 min — a ~30×
-// reduction in the steady-state cost that measured at ~45 % of all Redis CPU —
-// while keeping the self-healing property that made the naive loop safe.
-func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
-	idx.refresh(stor)
-	ticker := time.NewTicker(searchIndexRefreshInterval)
-	defer ticker.Stop()
-	ticks := 0
-	for range ticker.C {
-		ticks++
-		if ticks%searchIndexReconcileEvery == 0 {
-			idx.refresh(stor)
-			continue
-		}
-		// Drain until caught up, so a burst larger than one batch does not take
-		// one interval per batch to land.
-		for idx.refreshIncremental(stor) {
-		}
-	}
-}
-
-// serveIndexedSearch answers a search from the in-memory index: a token-AND
-// match against each record's haystack (or a whole-query hashID substring,
-// preserving the legacy behaviour) AND every structured filter, then returns
-// the full metadata for the <=limit survivors. No Redis round-trip per scanned
-// record — matching and results both come from memory.
+// Until 2026-10 this ran against an in-memory copy of every record, rebuilt
+// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh)
+// and is gone. The matchers did not change, so neither did the semantics.
 //
 // This is the ONLY search path. Free text, filters, both, or neither all land
 // here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
 func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
-	tuples, ready := s.search.snapshot()
-	if !ready {
-		// First call before the background build finished: warm inline with the
-		// efficient batched read (seconds), never the per-record SCAN fallback.
-		s.search.refresh(s.storage)
-		tuples, _ = s.search.snapshot()
-	}
 	tokens := strings.Fields(strings.ToLower(req.Query))
 	queryLower := strings.ToLower(req.Query)
 	filters := normalizedFilters(req)
-	results := make([]MetadataSearchResult, 0, req.Limit)
-	for _, t := range tuples {
-		if len(results) >= req.Limit {
-			break
+
+	narrow := make([]storage.SearchFilter, 0, len(filters))
+	for _, f := range filters {
+		narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
 	}
+	hits, err := s.storage.Search(req.Query, narrow, req.Limit, func(hashID string, fields map[string]string) bool {
 		if req.Query != "" &&
-			!matchAllTokens(t.Haystack, tokens) &&
-			!strings.Contains(strings.ToLower(t.HashID), queryLower) {
-			continue
+			!matchAllTokens(storage.Haystack(fields), tokens) &&
+			!strings.Contains(strings.ToLower(hashID), queryLower) {
+			return false
 		}
-		if !recordMatchesFilters(t.Fields, filters) {
-			continue
+		return recordMatchesFilters(fields, filters)
+	})
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, "search failed: "+err.Error())
+		return
 	}
-		// Full metadata is cached in the index — return it straight from memory,
-		// no Redis round-trip per result.
-		results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
+	total, err := s.storage.CountRoots()
+	if err != nil {
+		total = 0
+	}
+
+	results := make([]MetadataSearchResult, 0, len(hits))
+	for _, h := range hits {
+		results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields})
 	}
 	writeJSON(w, http.StatusOK, MetadataSearchResponse{
 		Results: results,
 		Count:   len(results),
-		Total:   len(tuples),
+		Total:   int(total),
 	})
 }
 

internal/api/server.go

@@ -9,10 +9,10 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/discovery"
 	"github.com/metazla/meta-core/internal/events"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/leader"
+	"github.com/metazla/meta-core/internal/meshdisco"
 	"github.com/metazla/meta-core/internal/mounts"
 	"github.com/metazla/meta-core/internal/schema"
 	"github.com/metazla/meta-core/internal/storage"
@@ -25,8 +25,6 @@ import (
 type Server struct {
 	config            *config.Config
 	leaderProvider    *leader.LeaderInfoProvider
-	discovery         *discovery.Service
-	cleaner           *discovery.Cleaner
 	storage           *storage.Client
 	mountsManager     *mounts.Manager
 	mountsHandlers    *mounts.Handlers
@@ -41,37 +39,46 @@ type Server struct {
 	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
-	search            *searchIndex
 	// challenges backs proof-of-possession on reveal/delete. In-process, so a
 	// restart invalidates the ones in flight — see identity/challenge.go.
 	challenges *identity.ChallengeStore
+	// mesh is the UDP discovery node (beacon v2). nil when discovery
+	// is disabled or failed to bind, in which case /api/neighbors reports an
+	// empty list rather than erroring — a dashboard nav is decoration and
+	// must not take the API down with it.
+	mesh   *meshdisco.Node
 	router *mux.Router
 	server *http.Server
+	// stats caches the keyspace-walk half of GET /api/stats so a polling
+	// dashboard never puts a multi-second store walk on the request path.
+	stats *statsCache
+}
+
+// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
+// server's constructor signature (and its tests) stay untouched.
+func (s *Server) SetMeshNode(n *meshdisco.Node) {
+	s.mesh = n
 }
 
 // NewServer creates a new API server
 func NewServer(
 	cfg *config.Config,
 	leaderProvider *leader.LeaderInfoProvider,
-	disc *discovery.Service,
-	cleaner *discovery.Cleaner,
 	stor *storage.Client,
 ) *Server {
 	s := &Server{
 		config:         cfg,
 		leaderProvider: leaderProvider,
-		discovery:      disc,
-		cleaner:        cleaner,
 		storage:        stor,
-		search:         &searchIndex{},
 		router:         mux.NewRouter(),
+		stats:          &statsCache{},
 	}
 
 	// Initialize file watcher/scanner and dispatcher
 	var scanFunc mounts.ScanFunc
 	if cfg.EnableFileWatcher {
 		s.watcherDispatcher = watcher.NewDispatcher()
-		// Wire storage client for Redis Stream publishing
+		// Wire storage client for file:events publishing
 		if stor != nil {
 			s.watcherDispatcher.SetStorageClient(stor)
 		}
@@ -81,7 +88,7 @@ func NewServer(
 		} else {
 			s.fileWatcher = fileWatcher
 			// Wire storage so the watcher can persist (path,size,mtime,cid)
-			// tuples to Redis as it computes midhashes.
+			// tuples to the store as it computes midhashes.
 			if stor != nil {
 				fileWatcher.SetStorageClient(stor)
 			}
@@ -103,7 +110,7 @@ func NewServer(
 				// Pass republish callback to handlers (handles nil metaPublisher case)
 				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
 
-				// Hydrate the in-memory state registry from Redis tuples so
+				// Hydrate the in-memory state registry from stored tuples so
 				// the first scan tick after boot can skip ComputeMidHash256
 				// for files whose (size, mtime) match.
 				configs := watchersManager.List()
@@ -114,7 +121,7 @@ func NewServer(
 				if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
 					log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
 				} else {
-					log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
+					log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
 				}
 			}
 		}
@@ -164,6 +171,10 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
 	s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
 
+	// Dashboard headline counters in one request. Cheap half live, keyspace
+	// walks served from a background-refreshed cache — see stats.go.
+	s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
+
 	// Metadata Editor API routes (must be before /meta/{hash} routes)
 	s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
 	s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
@@ -216,6 +227,8 @@ func (s *Server) setupRoutes() {
 	s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
 	s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
 	s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
+	s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
 	s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
 	s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
@@ -257,30 +270,17 @@ func (s *Server) setupRoutes() {
 	// without going through Authelia.
 	s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
 
-	// Admin: reunify stranded midhash-rooted entries into their UUID roots
-	// (the dual-root pattern caused by historical writes that bypassed CID
-	// resolution).
-	s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
-	// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
-	// (METADATA_KEYS.md §14.17). Idempotent.
-	s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
-	// One-shot sweep of query echoes on literature cards (meta-read
-	// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
-	s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
-
-	// SSE event streams. Mediated mirror of the Redis Streams; consumers
-	// reach them over HTTP instead of touching Redis directly. Auth-bypass
+	// SSE event streams. The only way consumers see meta-core's change
+	// streams — they never touch the database. Auth-bypass
 	// applies but exposure should be inside-only (do NOT proxy through
 	// Caddy publicly). See docs/api-mediated-access.md "Auth".
 	s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
 	s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
 
-	// Service discovery
-	// Both /services and /api/services are supported for consistency with other services
-	s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
-	s.router.HandleFunc("/api/services", s.handleListServices).Methods("GET")
-	s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
-	s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
+	// beacon v2: neighbours heard over UDP. Supersedes /api/services,
+	// which is still served above while consumers migrate.
+	s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
+	s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
 
 	// Mount management routes (if manager initialized)
 	if s.mountsHandlers != nil {
@@ -349,44 +349,31 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
-	// Build + periodically refresh the in-memory search index so
-	// POST /api/metadata/search matches in memory instead of a Redis
-	// round-trip per record (see search_index.go).
-	if s.storage != nil {
-		go s.search.runRefreshLoop(s.storage)
-	}
-
-	// Start meta publisher (if storage is connected)
-	// Publishes keyspace notifications to meta:events stream
+	// Start meta publisher (if storage is connected). Live change events are
+	// written by the storage engine itself; the publisher owns the bulk
+	// republish below.
 	if s.storage != nil && s.storage.IsConnected() {
-		s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient())
+		s.metaPublisher = events.NewMetaPublisher(s.storage)
 		if err := s.metaPublisher.Start(); err != nil {
 			log.Printf("[API] Warning: failed to start meta publisher: %v", err)
 		} else {
-			// Schema indexer consumes the same stream to derive a live
-			// per-field schema (type/hint/breakdown). Start it before the
-			// republish flood so it captures the bootstrap events.
-			s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
+			// Schema indexer: per-field schema (type/hint/breakdown) from a
+			// periodic scan of the store.
+			s.schemaIndexer = schema.NewIndexer(s.storage.Store())
 			if err := s.schemaIndexer.Start(); err != nil {
 				log.Printf("[API] Warning: failed to start schema indexer: %v", err)
 				s.schemaIndexer = nil
 			}
 
-			// Republish events for all existing metadata
+			// Republish events for all existing metadata. Consumers that
+			// rebuild by replaying meta:events from the beginning (meta-fuse)
+			// depend on it.
 			go func() {
-				// Build any missing per-record field indexes first, in ONE
-				// keyspace pass. The republish below reads every record, and
-				// an un-indexed record read costs a full-keyspace SCAN — on a
-				// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
-				// paid again on every restart). Warming first turns the whole
-				// republish into index-served reads. Idempotent, so a
-				// fully-indexed store just pays the single scan.
-				// See storage/field_index.go.
 				started := time.Now()
-				if n, err := s.storage.WarmFieldIndexes(); err != nil {
-					log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
+				if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
+					log.Printf("[API] Warning: search row rebuild failed: %v", err)
 				} else if n > 0 {
-					log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
+					log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
 				}
 
 				count, err := s.RepublishMetadata()

internal/api/snapshot.go

@@ -103,7 +103,7 @@ func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, result)
 }
 
-// handleSnapshotWipe clears Redis metadata.
+// handleSnapshotWipe clears all stored metadata.
 // POST /api/snapshot/wipe?scope=metadata
 func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {

internal/api/sse_events.go

@@ -3,7 +3,6 @@ package api
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"net/http"
@@ -11,15 +10,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
-// SSE event stream endpoints for the file:events and meta:events Redis Streams.
+// SSE event stream endpoints for the file:events and meta:events streams.
 //
 // Wire contract (see docs/api-mediated-access.md):
-//   - One SSE event per Redis Stream entry.
-//   - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
-//     MUST treat it as a black-box string they echo back via Last-Event-ID.
+//   - One SSE event per stream entry.
+//   - `id:` is the opaque stream entry ID (`<ms>-<seq>`, the Redis Streams
+//     form, kept across the move to SQLite so cursors consumers persisted to
+//     disk still compare correctly); consumers MUST treat it as a black-box
+//     string they echo back via Last-Event-ID.
 //   - `event:` is the `type` field from the underlying entry.
 //   - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
 //   - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
@@ -30,30 +31,30 @@ import (
 const (
 	sseHeartbeatInterval = 30 * time.Second
 
-	// sseReadBlock controls how long each XREAD blocks before returning.
+	// sseReadBlock controls how long each read waits for new entries.
 	// Much shorter than sseHeartbeatInterval so the handler stays responsive
 	// to client disconnects and the heartbeat check runs frequently enough
 	// that the keep-alive fires within a few seconds of its nominal cadence.
 	sseReadBlock = 2 * time.Second
 
-	// sseReadCount caps how many entries one XREAD returns. Small enough
+	// sseReadCount caps how many entries one read returns. Small enough
 	// to keep per-flush memory low, large enough to drain bursts cheaply.
 	sseReadCount = 100
 )
 
-// handleEventsFiles streams the file:events Redis Stream as SSE.
+// handleEventsFiles streams the file:events stream as SSE.
 func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "file:events")
 }
 
-// handleEventsMeta streams the meta:events Redis Stream as SSE.
+// handleEventsMeta streams the meta:events stream as SSE.
 func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
 	s.streamSSE(w, r, "meta:events")
 }
 
-// streamSSE is the shared SSE pump shared by both endpoints. One handler per
-// stream — there is no fan-out; each subscriber holds its own XREAD loop.
-// That's fine at our scale (≤20 subscribers; one per consumer service).
+// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads
+// the stream itself and sleeps on the store's commit broadcast between reads —
+// fine at our scale (≤20 subscribers; one per consumer service).
 func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
 	if !s.storage.IsConnected() {
 		w.Header().Set("Retry-After", "5")
@@ -89,14 +90,14 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 	w.WriteHeader(http.StatusOK)
 	flusher.Flush()
 
-	client := s.storage.GetRedisClient()
-	if client == nil {
+	store := s.storage.Store()
+	if store == nil {
 		// Storage was just disconnected — close cleanly.
 		return
 	}
 
 	ctx := r.Context()
-	cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
+	cursor := resolveStartCursor(ctx, store, stream, r.Header.Get("Last-Event-ID"), w, flusher)
 
 	lastWrite := time.Now()
 	for {
@@ -104,37 +105,34 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 			return
 		}
 
-		// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
-		// context's deadline isn't propagated into XREAD's BLOCK arg —
-		// that's a poll-style timeout on Redis itself — so we keep BLOCK
-		// short.
-		readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
-		result, err := client.XRead(readCtx, &redis.XReadArgs{
-			Streams: []string{stream, cursor},
-			Block:   sseReadBlock,
-			Count:   sseReadCount,
-		}).Result()
-		cancel()
-
+		// Take the wake-up channel BEFORE reading, so an append landing
+		// between the read and the wait still wakes us.
+		wake := store.Wait(stream)
+		entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount)
 		if err != nil {
-			// redis.Nil = BLOCK timeout with no entries. Anything else
-			// while the client is still connected is a real failure.
-			if errors.Is(err, redis.Nil) || isContextTimeout(err) {
-				// fall through to heartbeat check
-			} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
-				return
-			} else {
-				log.Printf("[SSE %s] XREAD error: %v", stream, err)
+			if ctx.Err() != nil {
 				return
 			}
+			log.Printf("[SSE %s] read error: %v", stream, err)
+			return
 		}
 
-		for _, str := range result {
-			for _, msg := range str.Messages {
-				writeSSEEntry(w, flusher, msg)
-				cursor = msg.ID
+		for _, e := range entries {
+			writeSSEEntry(w, flusher, e)
+			cursor = e.ID
 			lastWrite = time.Now()
 		}
+
+		if len(entries) == 0 {
+			timer := time.NewTimer(sseReadBlock)
+			select {
+			case <-ctx.Done():
+				timer.Stop()
+				return
+			case <-wake:
+			case <-timer.C:
+			}
+			timer.Stop()
 		}
 
 		// Heartbeat if quiet for >= the interval. Comment line; SSE
@@ -150,38 +148,44 @@ func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string
 }
 
 // resolveStartCursor decides where to start reading.
-//   - No Last-Event-ID → start at "$" (only new entries).
+//   - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
+//     entries appended from now on). Redis' "$" was re-resolved on every
+//     XREAD, which dropped anything appended between two reads.
 //   - Last-Event-ID provided AND still in stream → resume from it.
 //   - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
 //     and resume from the oldest available entry's predecessor "0", so the
-//     next XREAD picks up everything still in the stream.
+//     next read picks up everything still in the stream.
 //
-// The "still in stream" check is cheap: one XRANGE for the first entry.
-func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
-	if lastID == "" {
-		return "$"
-	}
-
-	// One-entry XRANGE to find the oldest ID currently in the stream.
+// The "still in stream" check is cheap: one read of the first entry.
+func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
 	xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
-	entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
+
+	if lastID == "" {
+		head, err := store.LastID(xCtx, stream)
+		if err != nil {
+			log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err)
+			return "0"
+		}
+		return head
+	}
+
+	oldestEntry, ok, err := store.Oldest(xCtx, stream)
 	if err != nil {
 		// Treat any error as "stream not available right now" and resume
-		// from the cursor — XREAD will either find it or block until new
-		// entries arrive.
-		log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
+		// from the cursor — the read either finds it or waits for new entries.
+		log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err)
 		return lastID
 	}
 
-	if len(entries) == 0 {
+	if !ok {
 		// Stream is empty. Cursor may be from a previous incarnation; safest
-		// is to resume from the cursor — XREAD against an empty stream
-		// returns nothing and blocks until new entries arrive.
+		// is to resume from the cursor — reading an empty stream returns
+		// nothing and waits until new entries arrive.
 		return lastID
 	}
 
-	oldest := entries[0].ID
+	oldest := oldestEntry.ID
 	if compareStreamIDs(lastID, oldest) >= 0 {
 		// Cursor is at or past the oldest — fine to resume.
 		return lastID
@@ -197,39 +201,35 @@ func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastI
 	fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
 	flusher.Flush()
 
-	// Return "0" so the next XREAD picks up entries starting from the
+	// Return "0" so the next read picks up entries starting from the
 	// stream's true minimum — including `oldest` itself.
 	return "0"
 }
 
-// writeSSEEntry serialises one Redis Stream entry as an SSE event.
+// writeSSEEntry serialises one stream entry as an SSE event.
 //
 // `event:` comes from the entry's `type` field (no prefix munging). `data:`
-// is a JSON object containing all OTHER fields. Field values from the Redis
-// stream are interface{} (typically strings); we encode whatever the stream
-// holds.
-func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
+// is a JSON object containing all OTHER fields.
+func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) {
 	eventType := "message"
-	data := make(map[string]interface{}, len(msg.Values))
-	for k, v := range msg.Values {
+	data := make(map[string]string, len(e.Values))
+	for k, v := range e.Values {
 		if k == "type" {
-			if s, ok := v.(string); ok {
-				eventType = s
+			eventType = v
 			continue
 		}
-		}
 		data[k] = v
 	}
 
 	payload, err := json.Marshal(data)
 	if err != nil {
-		log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
+		log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err)
 		return
 	}
 
 	// Spec: each event ends with a blank line. id/event/data lines are
 	// terminated with single \n.
-	fmt.Fprintf(w, "id: %s\n", msg.ID)
+	fmt.Fprintf(w, "id: %s\n", e.ID)
 	fmt.Fprintf(w, "event: %s\n", eventType)
 	fmt.Fprintf(w, "data: %s\n\n", string(payload))
 	flusher.Flush()
@@ -263,18 +263,3 @@ func parseStreamID(id string) (int64, int64) {
 	}
 	return ms, seq
 }
-
-// isContextTimeout returns true for the context deadline error wrapped by
-// go-redis when XREAD's read context fires. We treat it the same as
-// redis.Nil — i.e. an empty poll cycle, not a fatal error.
-func isContextTimeout(err error) bool {
-	if err == nil {
-		return false
-	}
-	if errors.Is(err, context.DeadlineExceeded) {
-		return true
-	}
-	// go-redis sometimes wraps this in its own error type. String match is
-	// a safety net.
-	return strings.Contains(err.Error(), "context deadline exceeded")
-}

internal/api/stats.go

@@ -0,0 +1,166 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/metazla/meta-core/internal/identity"
+)
+
+// statsSweepTTL is how long a keyspace-walk result stays good. The walk is
+// never on the request path (see statsCache), so this only controls how stale
+// the library tiles may look, not how slow the endpoint is.
+const statsSweepTTL = 60 * time.Second
+
+// fileStatsJSON is the file-backed half of the library: the roots that carry
+// filePath+sizeByte+mtimeNano. Gateway / cid-rooted records have no file on
+// disk and are counted in Records, not here — the gap between the two numbers
+// is exactly "metadata we hold for files we do not have".
+type fileStatsJSON struct {
+	Count     int   `json:"count"`
+	TotalSize int64 `json:"totalSize"`
+}
+
+// StatsResponse is the body of GET /api/stats — every headline number the
+// dashboard shows, in one request.
+//
+// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
+// cheap (an index range count, a cached row count, a file stat) plus a
+// directory listing, recomputed on every call. The `redis*` JSON names
+// predate the move to SQLite and are kept for dashboard compatibility:
+// redisKeys is the store's row count, redisMemory its on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
+// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
+// from a background-refreshed cache and are null until the first sweep lands.
+type StatsResponse struct {
+	Records     int64          `json:"records"`
+	RedisKeys   int64          `json:"redisKeys"`
+	RedisMemory string         `json:"redisMemory"`
+	Identities  int            `json:"identities"`
+	Files       *fileStatsJSON `json:"files"`
+	UDLUsers    *int           `json:"udlUsers"`
+	// SweptAt is when the cached half was computed (unix ms), 0 if never.
+	SweptAt int64 `json:"sweptAt"`
+	// Sweeping reports that a refresh is in flight, so the UI can show the
+	// cached numbers as provisional instead of pretending they are live.
+	Sweeping bool `json:"sweeping"`
+}
+
+// statsCache holds the expensive half of /api/stats.
+//
+// Why a cache and not just "compute it": the dashboard polls. A multi-second store walk
+// per tick, per open tab, is a self-inflicted outage on a box that is also
+// serving playback — the same shape as meta-sort's /api/stats full sweep. So
+// the handler NEVER waits on a walk: it returns whatever the last sweep
+// produced and kicks off a new one if the old one is stale. `running` makes
+// that single-flight, so ten tabs cost one walk.
+type statsCache struct {
+	mu       sync.Mutex
+	files    *fileStatsJSON
+	udlUsers *int
+	sweptAt  time.Time
+	running  bool
+}
+
+// snapshot returns the cached values and whether a refresh is in flight.
+func (c *statsCache) snapshot() (*fileStatsJSON, *int, time.Time, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.files, c.udlUsers, c.sweptAt, c.running
+}
+
+// refreshIfStale starts a background sweep unless one is already running or
+// the last result is still fresh. Returns true if it started one.
+func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
+	c.mu.Lock()
+	if c.running || time.Since(c.sweptAt) < statsSweepTTL {
+		c.mu.Unlock()
+		return false
+	}
+	c.running = true
+	c.mu.Unlock()
+
+	go func() {
+		files, udl := sweep()
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		c.running = false
+		// A failed sweep leaves the previous numbers in place rather than
+		// blanking the tiles; sweptAt still advances so a hard-failing walk
+		// (store down) does not spin once per poll.
+		c.sweptAt = time.Now()
+		if files != nil {
+			c.files = files
+		}
+		if udl != nil {
+			c.udlUsers = udl
+		}
+	}()
+	return true
+}
+
+// handleStats handles GET /api/stats.
+func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+
+	resp := StatsResponse{}
+
+	if n, err := s.storage.CountRoots(); err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	} else {
+		resp.Records = n
+	}
+	if n, err := s.storage.CountKeys(); err == nil {
+		resp.RedisKeys = n
+	}
+	if mem, err := s.storage.GetMemoryInfo(); err == nil {
+		resp.RedisMemory = mem
+	} else {
+		resp.RedisMemory = "N/A"
+	}
+	if list, err := identity.List(s.config.IdentityAccountsDir()); err == nil {
+		resp.Identities = len(list)
+	}
+
+	s.stats.refreshIfStale(s.sweepStats)
+	files, udl, sweptAt, sweeping := s.stats.snapshot()
+	resp.Files = files
+	resp.UDLUsers = udl
+	resp.Sweeping = sweeping
+	if !sweptAt.IsZero() {
+		resp.SweptAt = sweptAt.UnixMilli()
+	}
+
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// sweepStats runs the two keyspace walks behind the cached tiles. Called off
+// the request path only. A failing half returns nil so the other half still
+// lands.
+func (s *Server) sweepStats() (*fileStatsJSON, *int) {
+	var files *fileStatsJSON
+	if tuples, err := s.storage.GetTuplesForAllFiles(); err != nil {
+		log.Printf("[API] stats: file tuple sweep failed: %v", err)
+	} else {
+		fs := fileStatsJSON{Count: len(tuples)}
+		for _, t := range tuples {
+			fs.TotalSize += t.Size
+		}
+		files = &fs
+	}
+
+	var udl *int
+	if stats, err := s.storage.UDLAllUserStats(); err != nil {
+		log.Printf("[API] stats: UDL user sweep failed: %v", err)
+	} else {
+		n := len(stats)
+		udl = &n
+	}
+
+	return files, udl
+}

internal/api/storage_helper_test.go

@@ -0,0 +1,19 @@
+package api
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// newTestStorage opens a fresh metadata store in the test's temp dir.
+func newTestStorage(t *testing.T) *storage.Client {
+	t.Helper()
+	c := storage.NewClient()
+	if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	t.Cleanup(func() { _ = c.Close() })
+	return c
+}

internal/api/udl.go

@@ -7,12 +7,13 @@ package api
 // meta-watch's local op-log. Records arrive already signed (and, for
 // private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
 // version-gates and indexes them but never inspects the crypto. See
-// internal/storage/udl.go for the Redis model.
+// internal/storage/udl.go for the key model.
 
 import (
 	"encoding/json"
 	"math"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
@@ -64,7 +65,7 @@ type UDLPutBatchResponse struct {
 	Results []UDLPutBatchResult `json:"results"`
 }
 
-// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
+// maxUDLBatch bounds one batch. Each record is a store write while the
 // request is held open; sized to match the sign batch on the other side of the
 // same operation (see maxSignBatch).
 const maxUDLBatch = 1000
@@ -125,7 +126,7 @@ func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
 		Exists:  exists,
 	}
 	// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
-	// Redis, a future writer bug) would otherwise fail mid-encode and emit a
+	// the store, a future writer bug) would otherwise fail mid-encode and emit a
 	// truncated body instead of a readable error.
 	if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
 		raw := json.RawMessage(cell.Value)
@@ -271,6 +272,75 @@ func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
 }
 
+// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
+// — every record a uid owns, for cross-instance profile recovery.
+//
+// The other user-scoped reads answer a question with one coordinate already
+// known. Recovery knows only the uid: a box that has just imported a secret key
+// has no idea which cids or keys the profile ever touched, so it needs the
+// whole set or it gets nothing.
+//
+// Paged on a cid boundary — `nextCid` empty means the export is complete. The
+// response is opaque signed records, exactly as stored; whoever consumes it is
+// responsible for verifying each one before trusting it (meta-core has never
+// held the account key and cannot verify on anyone's behalf).
+func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	limit := maxUDLBatch
+	if v := r.URL.Query().Get("limit"); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil || n <= 0 {
+			writeError(w, http.StatusBadRequest, "limit must be a positive integer")
+			return
+		}
+		if n < limit {
+			limit = n
+		}
+	}
+	entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{
+		"entries": entries,
+		"nextCid": nextCid,
+	})
+}
+
+// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
+// uid holds records for.
+//
+// Exists so a gateway mirror can enforce a per-profile quota without walking
+// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
+// disk-fill DoS that needs no account. A quota check that itself cost a scan
+// would just move the denial of service one layer up.
+func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
+	if !s.storage.IsConnected() {
+		writeError(w, http.StatusServiceUnavailable, "storage not connected")
+		return
+	}
+	uid := mux.Vars(r)["uid"]
+	if uid == "" {
+		writeError(w, http.StatusBadRequest, "uid is required")
+		return
+	}
+	n, err := s.storage.UDLCountUserCids(uid)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
+}
+
 // handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
 func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
 	if !s.storage.IsConnected() {

internal/api/udl_test.go

@@ -6,9 +6,6 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
-
-	"github.com/alicebob/miniredis/v2"
-	"github.com/metazla/meta-core/internal/storage"
 )
 
 // newUDLTestServer returns a Server wired to an in-memory Redis.
@@ -18,14 +15,7 @@ import (
 // which the UDL handlers touch — they use s.storage and nothing else.
 func newUDLTestServer(t *testing.T) *Server {
 	t.Helper()
-
-	mr := miniredis.RunT(t)
-	c := storage.NewClient("")
-	if err := c.Connect("redis://" + mr.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
-	}
-	t.Cleanup(func() { _ = c.Close() })
-	return &Server{storage: c}
+	return &Server{storage: newTestStorage(t)}
 }
 
 // getRecord drives handleUDLRecordGet and decodes the body into raw JSON

internal/cid/cid-rank-vectors.json

@@ -134,6 +134,12 @@
       "rank": 5,
       "note": "codec 0x100A + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id - here (\"opensubtitles\",\"file:7061834\"). The GENERIC fetchable locator: one codec for every provider whose bytes are fetched the same way; the provider is inside the digest. Framing byte-identical to card (0x1007), but the byte path RESOLVES it (via source's fetch tier) instead of 404ing. Same codec-slot rule as the rest of the family: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5: once the fetched bytes' sha2-256 is linked onto the record it wins the canonical election with no addressing change."
     },
+    {
+      "name": "provider-file-tachiyomi",
+      "cid": "bagfcaabebf2gcy3inf4w63ljnvwhu332gnvw423po42hg33ug4zdeytdonrtk4bsnq",
+      "rank": 5,
+      "note": "provider-file (0x100A) under the second registered source, (\"tachiyomi\", base32lower(sha256(\"https://www.mangakakalot.gg/manga/ragna-crimson/chapter-94\"))[..26] = \"mlzoz3knkow4sot722bcsc5p2l\"). A comic chapter, minted by meta-feeder-suwayomi from the chapter's address ON ITS SITE - never the Suwayomi host/port/row ids - so every box mints this same cid; redeemed into one CBZ. Same rank as any provider-file (rank ignores source); the vector proves the digest bytes (cid-formats.md section 8.3)."
+    },
     {
       "name": "sha1",
       "cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",

internal/config/config.go

@@ -19,8 +19,8 @@ type Config struct {
 	BaseURL        string // Base URL for stable service discovery
 	PublicURL      string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
 
-	// Redis configuration
-	RedisPort int // Redis port (default: 6379)
+	// Storage configuration
+	DBPathOverride string // META_CORE_DB_PATH; empty = <MetaCorePath>/db/meta-core.db
 
 	// HTTP API configuration
 	HTTPPort int    // HTTP API port (default: 9000)
@@ -40,6 +40,14 @@ type Config struct {
 
 	// Mount configuration
 	MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
+
+	// UDP discovery (beacon v2). Replaces the file-based registry that used
+	// to live under /meta-core/services; see
+	// docs/project-architecture/beacon-v2.md.
+	EnableUDPDiscovery  bool   // Advertise + listen on the multicast group (default: true)
+	DiscoveryGroup      string // BEACON_GROUP: IPv4 multicast group (default: 239.255.99.1)
+	DiscoveryPort       int    // BEACON_PORT: UDP port (default: 9099)
+	DiscoveryIntervalMS int    // BEACON_INTERVAL_MS: unsolicited advertise interval (default: 10000)
 }
 
 // Load creates a Config from environment variables
@@ -52,7 +60,7 @@ func Load() *Config {
 		APIPort:                getEnvInt("API_PORT", 8180),
 		BaseURL:                getEnv("BASE_URL", ""),
 		PublicURL:              getEnv("META_CORE_PUBLIC_URL", ""),
-		RedisPort:             getEnvInt("REDIS_PORT", 6379),
+		DBPathOverride:         getEnv("META_CORE_DB_PATH", ""),
 		HTTPPort:               getEnvInt("META_CORE_HTTP_PORT", 9000),
 		HTTPHost:               getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
 		HealthCheckIntervalMS:  getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
@@ -63,6 +71,10 @@ func Load() *Config {
 		WatchIntervalMS:        getEnvInt("WATCH_INTERVAL_MS", 1000),
 		DebounceMS:             getEnvInt("DEBOUNCE_MS", 30000),
 		EnableFileWatcher:      getEnvBool("ENABLE_FILE_WATCHER", true),
+		EnableUDPDiscovery:     getEnvBool("ENABLE_UDP_DISCOVERY", true),
+		DiscoveryGroup:         getEnv("BEACON_GROUP", "239.255.99.1"),
+		DiscoveryPort:          getEnvInt("BEACON_PORT", 9099),
+		DiscoveryIntervalMS:    getEnvInt("BEACON_INTERVAL_MS", 10000),
 	}
 
 	// Set mounts directory
@@ -71,24 +83,20 @@ func Load() *Config {
 	return cfg
 }
 
-// LockFilePath returns the path to the leader lock file
+// LockFilePath returns the path to the database-volume mutex (one meta-core
+// per database file). Held by
+// docker/leader-election.sh; nothing reads it, and it is NOT service
+// discovery — see docs/project-architecture/beacon-v2.md.
 func (c *Config) LockFilePath() string {
 	return c.MetaCorePath + "/locks/kv-leader.lock"
 }
 
-// InfoFilePath returns the path to the leader info file
-func (c *Config) InfoFilePath() string {
-	return c.MetaCorePath + "/locks/kv-leader.info"
-}
-
-// RedisDataDir returns the path to Redis data directory
-func (c *Config) RedisDataDir() string {
-	return c.MetaCorePath + "/db/redis"
-}
-
-// ServicesDir returns the path to services directory
-func (c *Config) ServicesDir() string {
-	return c.MetaCorePath + "/services"
+// DBPath returns the path of meta-core's SQLite database file.
+func (c *Config) DBPath() string {
+	if c.DBPathOverride != "" {
+		return c.DBPathOverride
+	}
+	return c.MetaCorePath + "/db/meta-core.db"
 }
 
 // MountsFilePath returns the path to the mounts configuration file

internal/discovery/cleaner.go

@@ -1,196 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"log"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// CleanupStats holds statistics about the cleanup process
-type CleanupStats struct {
-	LastRunTime     string `json:"lastRunTime,omitempty"`
-	ServicesRemoved int    `json:"servicesRemoved"`
-	TotalRuns       int    `json:"totalRuns"`
-	Errors          int    `json:"errors"`
-}
-
-// Cleaner handles periodic cleanup of dead service registration files
-type Cleaner struct {
-	config      *config.Config
-	servicesDir string
-	ownFile     string // Our own service file (never delete)
-
-	stats    CleanupStats
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-	mu       sync.RWMutex
-}
-
-// NewCleaner creates a new service cleaner
-func NewCleaner(cfg *config.Config) *Cleaner {
-	hostname, _ := os.Hostname()
-	return &Cleaner{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		ownFile:     filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins the cleanup loop
-func (c *Cleaner) Start() error {
-	log.Printf("[Cleaner] Starting dead service cleanup (interval: %dms, threshold: %dms)",
-		c.config.CleanupIntervalMS, c.config.DeadServiceThresholdMS)
-
-	c.wg.Add(1)
-	go c.cleanupLoop()
-
-	return nil
-}
-
-// Stop stops the cleanup loop
-func (c *Cleaner) Stop() error {
-	log.Println("[Cleaner] Stopping dead service cleanup...")
-	close(c.stopChan)
-	c.wg.Wait()
-	return nil
-}
-
-// cleanupLoop runs the periodic cleanup
-func (c *Cleaner) cleanupLoop() {
-	defer c.wg.Done()
-
-	// Run an initial cleanup shortly after starting
-	initialDelay := time.NewTimer(30 * time.Second)
-	select {
-	case <-c.stopChan:
-		initialDelay.Stop()
-		return
-	case <-initialDelay.C:
-		c.runCleanup()
-	}
-
-	ticker := time.NewTicker(time.Duration(c.config.CleanupIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-c.stopChan:
-			return
-		case <-ticker.C:
-			c.runCleanup()
-		}
-	}
-}
-
-// runCleanup scans the services directory and removes dead services
-func (c *Cleaner) runCleanup() {
-	c.mu.Lock()
-	c.stats.TotalRuns++
-	c.stats.LastRunTime = time.Now().UTC().Format(time.RFC3339)
-	c.mu.Unlock()
-
-	entries, err := os.ReadDir(c.servicesDir)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Printf("[Cleaner] Failed to read services directory: %v", err)
-			c.mu.Lock()
-			c.stats.Errors++
-			c.mu.Unlock()
-		}
-		return
-	}
-
-	threshold := time.Duration(c.config.DeadServiceThresholdMS) * time.Millisecond
-	removedCount := 0
-
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(c.servicesDir, entry.Name())
-
-		// Never delete our own service file
-		if filePath == c.ownFile {
-			continue
-		}
-
-		// Check if service is dead
-		isDead, serviceName, age := c.isServiceDead(filePath, threshold)
-		if isDead {
-			if err := os.Remove(filePath); err != nil {
-				log.Printf("[Cleaner] Failed to remove dead service file %s: %v", entry.Name(), err)
-				c.mu.Lock()
-				c.stats.Errors++
-				c.mu.Unlock()
-			} else {
-				ageMinutes := age.Minutes()
-				log.Printf("[Cleaner] Removed dead service: %s (last heartbeat: %.1f minutes ago)", serviceName, ageMinutes)
-				removedCount++
-			}
-		}
-	}
-
-	if removedCount > 0 {
-		c.mu.Lock()
-		c.stats.ServicesRemoved += removedCount
-		c.mu.Unlock()
-		log.Printf("[Cleaner] Cleanup complete: removed %d dead service(s)", removedCount)
-	}
-}
-
-// isServiceDead checks if a service file represents a dead service
-// Returns: isDead, serviceName, age since last heartbeat
-func (c *Cleaner) isServiceDead(filePath string, threshold time.Duration) (bool, string, time.Duration) {
-	data, err := os.ReadFile(filePath)
-	if err != nil {
-		// Can't read file - treat as dead to clean up corrupted files
-		log.Printf("[Cleaner] Cannot read service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, filepath.Base(filePath), threshold
-	}
-
-	var info ServiceInfo
-	if err := json.Unmarshal(data, &info); err != nil {
-		// Corrupted JSON - skip rather than delete (might be recoverable)
-		log.Printf("[Cleaner] Skipping corrupted service file %s: %v", filepath.Base(filePath), err)
-		return false, "", 0
-	}
-
-	// Parse last heartbeat timestamp
-	if info.LastHeartbeat == "" {
-		// No heartbeat timestamp - treat as dead
-		return true, info.Name, threshold
-	}
-
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err != nil {
-		// Invalid timestamp - treat as dead
-		log.Printf("[Cleaner] Invalid timestamp in service file %s: %v (treating as dead)", filepath.Base(filePath), err)
-		return true, info.Name, threshold
-	}
-
-	age := time.Since(lastHeartbeat)
-	if age > threshold {
-		return true, info.Name, age
-	}
-
-	return false, info.Name, age
-}
-
-// Stats returns a copy of the current cleanup statistics
-func (c *Cleaner) Stats() CleanupStats {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	return CleanupStats{
-		LastRunTime:     c.stats.LastRunTime,
-		ServicesRemoved: c.stats.ServicesRemoved,
-		TotalRuns:       c.stats.TotalRuns,
-		Errors:          c.stats.Errors,
-	}
-}

internal/discovery/service.go

@@ -1,286 +0,0 @@
-package discovery
-
-import (
-	"encoding/json"
-	"fmt"
-	"log"
-	"net"
-	"os"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"github.com/metazla/meta-core/internal/config"
-)
-
-// ServiceInfo contains simplified service registration data
-// URLs are obtained via the /urls API endpoint
-type ServiceInfo struct {
-	Name          string `json:"name"`
-	Hostname      string `json:"hostname"`
-	BaseUrl       string `json:"baseUrl"`
-	Status        string `json:"status"`
-	LastHeartbeat string `json:"lastHeartbeat"`
-}
-
-// Service handles service registration and discovery
-type Service struct {
-	config      *config.Config
-	servicesDir string
-	serviceFile string
-	info        *ServiceInfo
-	mu          sync.RWMutex
-
-	stopChan chan struct{}
-	wg       sync.WaitGroup
-}
-
-// NewService creates a new service discovery instance
-func NewService(cfg *config.Config) *Service {
-	hostname, _ := os.Hostname()
-	return &Service{
-		config:      cfg,
-		servicesDir: cfg.ServicesDir(),
-		serviceFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
-		stopChan:    make(chan struct{}),
-	}
-}
-
-// Start begins service registration and heartbeat
-func (s *Service) Start() error {
-	log.Printf("[Discovery] Starting service discovery for %s", s.config.ServiceName)
-
-	// Ensure services directory exists
-	if err := os.MkdirAll(s.servicesDir, 0755); err != nil {
-		return fmt.Errorf("failed to create services directory: %w", err)
-	}
-
-	// Build and register service info
-	s.info = s.buildServiceInfo()
-	if err := s.register(); err != nil {
-		return fmt.Errorf("failed to register service: %w", err)
-	}
-
-	// Start heartbeat loop
-	s.wg.Add(1)
-	go s.heartbeatLoop()
-
-	return nil
-}
-
-// Stop stops service discovery and unregisters
-func (s *Service) Stop() error {
-	log.Println("[Discovery] Stopping service discovery...")
-	close(s.stopChan)
-	s.wg.Wait()
-
-	// Unregister by removing service file
-	if err := os.Remove(s.serviceFile); err != nil && !os.IsNotExist(err) {
-		log.Printf("[Discovery] Failed to remove service file: %v", err)
-	}
-
-	return nil
-}
-
-// buildServiceInfo creates the service info for this instance
-func (s *Service) buildServiceInfo() *ServiceInfo {
-	hostname, _ := os.Hostname()
-	ip := getLocalIP()
-
-	// Browser-facing nav URL: prefer META_CORE_PUBLIC_URL (the reachable URL
-	// when there's no Caddy perimeter in front — e.g. a debug-direct port),
-	// then BASE_URL (the Caddy perimeter URL), then the internal IP.
-	var baseUrl string
-	if s.config.PublicURL != "" {
-		baseUrl = s.config.PublicURL
-	} else if s.config.BaseURL != "" {
-		baseUrl = s.config.BaseURL
-	} else {
-		baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
-	}
-
-	info := &ServiceInfo{
-		Name:          s.config.ServiceName,
-		Hostname:      hostname,
-		BaseUrl:       baseUrl,
-		Status:        "running",
-		LastHeartbeat: time.Now().UTC().Format(time.RFC3339),
-	}
-
-	return info
-}
-
-// register writes service info to file
-func (s *Service) register() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	return s.writeServiceInfo(s.info)
-}
-
-// writeServiceInfo atomically writes service info to file
-func (s *Service) writeServiceInfo(info *ServiceInfo) error {
-	tempPath := s.serviceFile + ".tmp"
-
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		return err
-	}
-
-	if err := os.WriteFile(tempPath, data, 0644); err != nil {
-		return err
-	}
-
-	return os.Rename(tempPath, s.serviceFile)
-}
-
-// heartbeatLoop periodically updates the last heartbeat
-func (s *Service) heartbeatLoop() {
-	defer s.wg.Done()
-
-	ticker := time.NewTicker(time.Duration(s.config.HeartbeatIntervalMS) * time.Millisecond)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-s.stopChan:
-			return
-		case <-ticker.C:
-			s.heartbeat()
-		}
-	}
-}
-
-// heartbeat updates the last heartbeat timestamp
-func (s *Service) heartbeat() {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return
-	}
-
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-
-	if err := s.writeServiceInfo(s.info); err != nil {
-		log.Printf("[Discovery] Failed to update heartbeat: %v", err)
-	}
-}
-
-// UpdateStatus updates the service status
-func (s *Service) UpdateStatus(status string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if s.info == nil {
-		return fmt.Errorf("service not registered")
-	}
-
-	s.info.Status = status
-	s.info.LastHeartbeat = time.Now().UTC().Format(time.RFC3339)
-	return s.writeServiceInfo(s.info)
-}
-
-// Discover finds a service by name
-// Looks for files matching pattern: name-*.json (hostname-based naming)
-func (s *Service) Discover(name string) (*ServiceInfo, error) {
-	// First try exact match for backward compatibility
-	exactPath := filepath.Join(s.servicesDir, name+".json")
-	if data, err := os.ReadFile(exactPath); err == nil {
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err == nil {
-			return s.checkStale(&info), nil
-		}
-	}
-
-	// Search for hostname-based files: name-*.json
-	pattern := filepath.Join(s.servicesDir, name+"-*.json")
-	matches, err := filepath.Glob(pattern)
-	if err != nil {
-		return nil, err
-	}
-
-	// Return the first valid service found
-	for _, match := range matches {
-		data, err := os.ReadFile(match)
-		if err != nil {
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			continue
-		}
-
-		return s.checkStale(&info), nil
-	}
-
-	return nil, nil
-}
-
-// checkStale marks a service as stale if heartbeat is too old
-func (s *Service) checkStale(info *ServiceInfo) *ServiceInfo {
-	lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
-	if err == nil {
-		staleThreshold := time.Duration(s.config.StaleThresholdMS) * time.Millisecond
-		if time.Since(lastHeartbeat) > staleThreshold {
-			info.Status = "stale"
-		}
-	}
-	return info
-}
-
-// DiscoverAll finds all registered services
-func (s *Service) DiscoverAll() ([]*ServiceInfo, error) {
-	entries, err := os.ReadDir(s.servicesDir)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return []*ServiceInfo{}, nil
-		}
-		return nil, err
-	}
-
-	var services []*ServiceInfo
-	for _, entry := range entries {
-		if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
-			continue
-		}
-
-		filePath := filepath.Join(s.servicesDir, entry.Name())
-		data, err := os.ReadFile(filePath)
-		if err != nil {
-			log.Printf("[Discovery] Failed to read service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		var info ServiceInfo
-		if err := json.Unmarshal(data, &info); err != nil {
-			log.Printf("[Discovery] Failed to parse service file %s: %v", entry.Name(), err)
-			continue
-		}
-
-		services = append(services, s.checkStale(&info))
-	}
-
-	return services, nil
-}
-
-// getLocalIP returns the local IP address
-func getLocalIP() string {
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		hostname, _ := os.Hostname()
-		return hostname
-	}
-
-	for _, addr := range addrs {
-		if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
-			if ipnet.IP.To4() != nil {
-				return ipnet.IP.String()
-			}
-		}
-	}
-
-	hostname, _ := os.Hostname()
-	return hostname
-}

internal/events/meta_publisher.go

@@ -1,199 +1,89 @@
 package events
 
 import (
-	"context"
 	"fmt"
 	"log"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/redis/go-redis/v9"
-
 	"github.com/metazla/meta-core/internal/storage"
 )
 
 const (
-	// MetaEventsStream is the Redis stream name for metadata events
-	MetaEventsStream = "meta:events"
+	// MetaEventsStream is the stream name for metadata events.
+	MetaEventsStream = storage.MetaEventsStream
 
 	// MetaEventsMaxLen is the approximate cap on stream length. Bounded
 	// retention is a contract requirement for the SSE wire — see
 	// docs/api-mediated-access.md "Backing-store retention and lifecycle".
-	// Cheap (~) trim keeps growth bounded without serialising writes.
-	MetaEventsMaxLen = 100_000
+	MetaEventsMaxLen = storage.MetaEventsMaxLen
 )
 
-// MetaPublisher subscribes to Redis keyspace notifications
-// and publishes metadata change events to the meta:events stream
+// republishBatch is how many events one republish transaction appends —
+// large enough to amortise the commit, small enough that the writer is never
+// held for long.
+const republishBatch = 5_000
+
+// MetaPublisher owns the meta:events stream's bulk republish.
+//
+// Live change events no longer pass through here: under Redis this subscribed
+// to keyspace notifications and copied each into the stream, a hop that could
+// drop events whenever the subscriber lagged. The storage engine now appends
+// the event in the same transaction as the change (internal/kv), so a change
+// and its event commit together.
 type MetaPublisher struct {
-	client  *redis.Client
-	ctx     context.Context
-	cancel  context.CancelFunc
-	wg      sync.WaitGroup
+	client  *storage.Client
 	running bool
 	mu      sync.RWMutex
 }
 
-// NewMetaPublisher creates a new MetaPublisher
-func NewMetaPublisher(client *redis.Client) *MetaPublisher {
-	ctx, cancel := context.WithCancel(context.Background())
-	return &MetaPublisher{
-		client: client,
-		ctx:    ctx,
-		cancel: cancel,
-	}
+// NewMetaPublisher creates a new MetaPublisher.
+func NewMetaPublisher(client *storage.Client) *MetaPublisher {
+	return &MetaPublisher{client: client}
 }
 
-// Start begins subscribing to keyspace notifications.
-//
-// The stream is NOT truncated on startup — the SSE wire contract requires
-// Last-Event-ID resumption to survive a meta-core restart, which means
-// entries must persist across restart. Growth is bounded instead by
-// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
+// Start marks the publisher ready. The stream is NOT truncated on startup —
+// the SSE wire contract requires Last-Event-ID resumption to survive a
+// meta-core restart. Growth is bounded by MetaEventsMaxLen instead.
 func (p *MetaPublisher) Start() error {
 	p.mu.Lock()
-	if p.running {
-		p.mu.Unlock()
-		return nil
-	}
+	defer p.mu.Unlock()
 	p.running = true
-	p.mu.Unlock()
-
-	p.wg.Add(1)
-	go p.subscribeLoop()
-
-	log.Println("[MetaPublisher] Started - listening for keyspace notifications")
+	log.Println("[MetaPublisher] Started")
 	return nil
 }
 
-// Stop stops the publisher
+// Stop stops the publisher.
 func (p *MetaPublisher) Stop() error {
 	p.mu.Lock()
-	if !p.running {
-		p.mu.Unlock()
-		return nil
-	}
+	defer p.mu.Unlock()
 	p.running = false
-	p.mu.Unlock()
-
-	p.cancel()
-	p.wg.Wait()
-
 	log.Println("[MetaPublisher] Stopped")
 	return nil
 }
 
-// IsRunning returns true if the publisher is running
+// IsRunning returns true if the publisher is running.
 func (p *MetaPublisher) IsRunning() bool {
 	p.mu.RLock()
 	defer p.mu.RUnlock()
 	return p.running
 }
 
-// subscribeLoop subscribes to keyspace notifications and publishes events
-func (p *MetaPublisher) subscribeLoop() {
-	defer p.wg.Done()
-
-	// Subscribe to all file:* keyspace events
-	// Pattern: __keyspace@0__:file:*
-	pattern := "__keyspace@0__:file:*"
-	pubsub := p.client.PSubscribe(p.ctx, pattern)
-	defer pubsub.Close()
-
-	log.Printf("[MetaPublisher] Subscribed to pattern: %s", pattern)
-
-	ch := pubsub.Channel()
-	for {
-		select {
-		case <-p.ctx.Done():
-			return
-		case msg := <-ch:
-			if msg == nil {
-				continue
-			}
-			// msg.Channel: __keyspace@0__:file:midhash256:abc123/tmdb
-			// msg.Payload: set, del, expire, etc.
-			p.publishEvent(msg.Channel, msg.Payload)
-		}
-	}
-}
-
-// publishEvent publishes a keyspace event to the meta:events stream
-func (p *MetaPublisher) publishEvent(channel, operation string) {
-	// Extract key from channel: __keyspace@0__:file:xxx/field -> file:xxx/field
-	key := strings.TrimPrefix(channel, "__keyspace@0__:")
-
-	// Skip index key and non-file keys
-	if strings.Contains(key, "__index__") || !strings.HasPrefix(key, "file:") {
-		return
-	}
-	// Skip the per-record field index. It is bookkeeping (see
-	// storage.FieldsField), not a metadata field — publishing it would make
-	// every downstream consumer that parses `file:<id>/<field>` off this
-	// stream (meta-search, meta-dup, meta-fuse, meta-sort) see a phantom
-	// field change on a key they can never read.
-	if strings.HasSuffix(key, "/"+storage.FieldsField) {
-		return
-	}
-
-	// Build event fields
-	fields := map[string]interface{}{
-		"type": operation, // "set", "del", "expire"
-		"key":  key,       // "file:midhash256:abc123/tmdb"
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	// Approximate MAXLEN trim keeps the stream bounded across restarts
-	// (see Start). Cap is intentionally generous; most consumers reconnect
-	// well within retention.
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing event: %v", err)
-	}
-}
-
-// publishEventDirect publishes an event directly (not from keyspace notification)
-func (p *MetaPublisher) publishEventDirect(operation, key string) {
-	fields := map[string]interface{}{
-		"type": operation,
-		"key":  key,
-		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
-	}
-
-	_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
-		Stream: MetaEventsStream,
-		MaxLen: MetaEventsMaxLen,
-		Approx: true,
-		Values: fields,
-	}).Result()
-
-	if err != nil {
-		log.Printf("[MetaPublisher] Error publishing direct event: %v", err)
-	}
-}
-
 // RepublishFunc is a function type for getting metadata that can be republished
 type RepublishFunc func() (hashIDs []string, getMetadata func(hashID string) (map[string]string, error), err error)
 
-// RepublishAllMetadata iterates over all existing file metadata and publishes events
-// This is called on startup and can be triggered via API
+// RepublishAllMetadata publishes a `set` event for every property of every
+// record. Called on startup and via the API.
+//
+// It exists for consumers that rebuild their state by replaying meta:events
+// from the beginning (meta-fuse's VFS does, with initialCursor 0-0) rather
+// than backfilling over HTTP. Appended in bounded batches, each its own
+// transaction, so a multi-million-event republish never holds the writer.
 func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int, error) {
-	p.mu.RLock()
-	running := p.running
-	p.mu.RUnlock()
-
-	if !running {
+	if !p.IsRunning() {
 		return 0, fmt.Errorf("publisher not running")
 	}
 
-	// Get all hash IDs and metadata getter
 	hashIDs, getMetadata, err := republishFunc()
 	if err != nil {
 		return 0, fmt.Errorf("failed to get hash IDs: %w", err)
@@ -201,22 +91,39 @@ func (p *MetaPublisher) RepublishAllMetadata(republishFunc RepublishFunc) (int,
 
 	log.Printf("[MetaPublisher] Republishing events for %d files...", len(hashIDs))
 
-	publishedCount := 0
+	published := 0
+	batch := make([]map[string]string, 0, republishBatch)
+	flush := func() error {
+		if err := p.client.AppendEvents(MetaEventsStream, MetaEventsMaxLen, batch); err != nil {
+			return err
+		}
+		published += len(batch)
+		batch = batch[:0]
+		return nil
+	}
 	for _, hashID := range hashIDs {
-		// Get all properties for this file
 		metadata, err := getMetadata(hashID)
 		if err != nil || metadata == nil {
 			continue
 		}
-
-		// Publish a set event for each property
+		ts := fmt.Sprintf("%d", time.Now().UnixMilli())
 		for property := range metadata {
-			key := fmt.Sprintf("file:%s/%s", hashID, property)
-			p.publishEventDirect("set", key)
-			publishedCount++
+			batch = append(batch, map[string]string{
+				"type": "set",
+				"key":  fmt.Sprintf("file:%s/%s", hashID, property),
+				"ts":   ts,
+			})
+			if len(batch) >= republishBatch {
+				if err := flush(); err != nil {
+					return published, fmt.Errorf("republish: %w", err)
 				}
 			}
+		}
+	}
+	if err := flush(); err != nil {
+		return published, fmt.Errorf("republish: %w", err)
+	}
 
-	log.Printf("[MetaPublisher] Republished %d property events", publishedCount)
-	return publishedCount, nil
+	log.Printf("[MetaPublisher] Republished %d property events", published)
+	return published, nil
 }

internal/identity/verify.go

@@ -56,7 +56,7 @@ func ValidAction(a Action) bool {
 }
 
 // ValidUID exposes the uid shape check to callers outside this package (the
-// API layer, which must reject a hostile uid before it reaches a Redis SCAN
+// API layer, which must reject a hostile uid before it reaches a key-range read
 // pattern or a file path).
 func ValidUID(uid string) bool { return validUID(uid) }
 

internal/kv/kv.go

@@ -0,0 +1,713 @@
+// Package kv is meta-core's storage engine: one string→string table in an
+// embedded SQLite file.
+//
+// # Why a key/value table and not a schema
+//
+// meta-core's data model is schema-free on purpose — a record is a set of flat
+// `file:<rid>/<field>` keys, and a new metadata field is just a new key
+// (METADATA_KEYS.md). The engine keeps exactly that model:
+//
+//	CREATE TABLE kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID
+//
+// is the only table that holds data, and it never needs a migration. What
+// Redis gave us that a bare KV would not — "list a record's fields", "find
+// records by field value", free-text search — comes from the table being
+// *ordered* (a record is a key range) and from two derived structures the
+// storage package maintains on top: an expression index over the field name
+// parsed out of the key, and an FTS5 table. Both are rebuildable from the kv
+// rows and never authoritative.
+//
+// It replaced Redis on 2026-10: holding everything in RAM cost ~6.6 GB on a
+// 169k-record corpus (Redis plus the API's in-memory copy needed to search
+// it), with the OOM kills that implies. The same data is ~2.5 GB on disk here
+// and the working set is a bounded page cache.
+//
+// # Concurrency
+//
+// SQLite allows one writer at a time, so every write goes through one writer
+// goroutine owning one connection. Concurrent Update calls are GROUP-COMMITTED:
+// the writer runs every request waiting at that moment inside one SQLite
+// transaction, each under its own SAVEPOINT (a failing request rolls back
+// alone), and commits once. A commit — writing the dirty pages of the table,
+// the field index and the FTS shadow tables to the WAL — was ~40% of the cost
+// of a record write, so sharing it is what makes concurrent ingest scale.
+//
+// Reads use a separate pool and see the last committed state (WAL). Inside
+// Update, read through the *Tx — the reader pool cannot see uncommitted rows —
+// and never call Update from inside an Update callback (the writer would wait
+// for itself).
+//
+// # Events
+//
+// Redis keyspace notifications are replaced by the transaction itself: every
+// Put/Delete of a key the Options.EventKey predicate selects is appended, in
+// the same transaction, to Options.EventStream as {type: set|del, key, ts}.
+// A change and its event therefore commit together or not at all.
+package kv
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	_ "modernc.org/sqlite"
+)
+
+// Options configures a Store.
+type Options struct {
+	// EventStream receives a set/del entry for every changed key EventKey
+	// selects. Empty disables change events.
+	EventStream string
+	// EventMaxLen is the approximate retention of EventStream.
+	EventMaxLen int64
+	// EventKey selects the keys whose changes are published.
+	EventKey func(key string) bool
+	// WriterCacheMB / ReaderCacheMB size SQLite's page cache per connection.
+	WriterCacheMB int
+	ReaderCacheMB int
+	// Readers is the size of the read connection pool.
+	Readers int
+	// Schema is extra DDL run at open (idempotent CREATE … IF NOT EXISTS),
+	// for derived indexes owned by the caller.
+	Schema string
+}
+
+// Store is an open database.
+type Store struct {
+	path string
+	opts Options
+	w    *sql.DB // single connection: the one writer
+	r    *sql.DB // read pool
+
+	streamMu sync.Mutex
+	streams  map[string]*streamState
+
+	// Hot write statements, prepared once on the writer connection: parsing
+	// SQL on every Put was the single largest cost of a record write.
+	stPut, stPutIfAbsent, stDelete, stGet, stInsert *sql.Stmt
+
+	writes    chan *writeReq
+	writeMu   sync.Mutex // held by the writer while a group runs
+	closeOnce sync.Once
+	stopped   chan struct{}
+}
+
+// writeReq is one Update call waiting for the writer.
+type writeReq struct {
+	fn   func(tx *Tx) error
+	done chan error
+}
+
+// maxGroup caps how many requests share one commit, bounding how long the
+// writer holds a transaction open.
+const maxGroup = 128
+
+// Open opens (creating if needed) the database file at path.
+func Open(path string, opts Options) (*Store, error) {
+	if opts.Readers <= 0 {
+		opts.Readers = 8
+	}
+	if opts.WriterCacheMB <= 0 {
+		opts.WriterCacheMB = 64
+	}
+	if opts.ReaderCacheMB <= 0 {
+		opts.ReaderCacheMB = 16
+	}
+	if opts.EventMaxLen <= 0 {
+		opts.EventMaxLen = 100_000
+	}
+
+	w, err := sql.Open("sqlite", dsn(path, opts.WriterCacheMB, false))
+	if err != nil {
+		return nil, fmt.Errorf("open writer: %w", err)
+	}
+	w.SetMaxOpenConns(1)
+	w.SetMaxIdleConns(1)
+	w.SetConnMaxLifetime(0)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if _, err := w.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID`); err != nil {
+		w.Close()
+		return nil, fmt.Errorf("create kv: %w", err)
+	}
+	if opts.Schema != "" {
+		// No deadline: on an existing store, the first open after a new index
+		// is added builds it over every row (minutes at millions of keys).
+		// Cancelling that just rolls it back and restarts it on the next boot.
+		started := time.Now()
+		if _, err := w.ExecContext(context.Background(), opts.Schema); err != nil {
+			w.Close()
+			return nil, fmt.Errorf("apply schema: %w", err)
+		}
+		if d := time.Since(started); d > 5*time.Second {
+			log.Printf("[kv] schema applied in %s (indexes built)", d.Round(time.Second))
+		}
+	}
+
+	r, err := sql.Open("sqlite", dsn(path, opts.ReaderCacheMB, true))
+	if err != nil {
+		w.Close()
+		return nil, fmt.Errorf("open readers: %w", err)
+	}
+	r.SetMaxOpenConns(opts.Readers)
+	r.SetMaxIdleConns(opts.Readers)
+
+	st := &Store{path: path, opts: opts, w: w, r: r, streams: make(map[string]*streamState)}
+	// Fresh deadline: the schema step above may legitimately have taken minutes.
+	ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	for _, p := range []struct {
+		dst   **sql.Stmt
+		query string
+	}{
+		{&st.stPut, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO UPDATE SET v = excluded.v`},
+		{&st.stPutIfAbsent, `INSERT INTO kv(k, v) VALUES(?, ?) ON CONFLICT(k) DO NOTHING`},
+		{&st.stDelete, `DELETE FROM kv WHERE k = ?`},
+		{&st.stGet, `SELECT v FROM kv WHERE k = ?`},
+		{&st.stInsert, `INSERT INTO kv(k, v) VALUES(?, ?)`},
+	} {
+		if *p.dst, err = w.PrepareContext(ctx, p.query); err != nil {
+			r.Close()
+			w.Close()
+			return nil, fmt.Errorf("prepare: %w", err)
+		}
+	}
+	st.writes = make(chan *writeReq)
+	st.stopped = make(chan struct{})
+	go st.writer()
+	return st, nil
+}
+
+// dsn builds a modernc DSN. WAL + synchronous=NORMAL is the durability class
+// Redis' AOF-everysec gave us: a power loss may drop the last transactions,
+// never corrupt the file.
+func dsn(path string, cacheMB int, readOnly bool) string {
+	q := url.Values{}
+	q.Add("_pragma", "busy_timeout(30000)")
+	q.Add("_pragma", "journal_mode(WAL)")
+	q.Add("_pragma", "synchronous(NORMAL)")
+	q.Add("_pragma", fmt.Sprintf("cache_size(-%d)", cacheMB*1024))
+	q.Add("_pragma", "temp_store(MEMORY)")
+	q.Add("_pragma", "journal_size_limit(268435456)")
+	if readOnly {
+		q.Add("_pragma", "query_only(1)")
+	} else {
+		q.Add("_txlock", "immediate")
+	}
+	return "file:" + path + "?" + q.Encode()
+}
+
+// Close stops the writer and closes both pools, checkpointing the WAL into the
+// main file.
+func (s *Store) Close() error {
+	s.closeOnce.Do(func() { close(s.stopped) })
+	s.writeMu.Lock() // wait for an in-flight group to finish
+	defer s.writeMu.Unlock()
+	_, _ = s.w.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`)
+	return errors.Join(s.r.Close(), s.w.Close())
+}
+
+// Ping reports whether the database answers.
+func (s *Store) Ping(ctx context.Context) error {
+	var one int
+	return s.r.QueryRowContext(ctx, `SELECT 1`).Scan(&one)
+}
+
+// Path is the database file path.
+func (s *Store) Path() string { return s.path }
+
+// DB exposes the read pool for callers that own derived indexes (search). It
+// is read-only (query_only), so it cannot bypass Update.
+func (s *Store) DB() *sql.DB { return s.r }
+
+// FileSize returns the on-disk size of the database plus its WAL.
+func (s *Store) FileSize() int64 {
+	var total int64
+	for _, suffix := range []string{"", "-wal"} {
+		if fi, err := os.Stat(s.path + suffix); err == nil {
+			total += fi.Size()
+		}
+	}
+	return total
+}
+
+// Checkpoint folds the WAL back into the main file. TRUNCATE takes the writer
+// slot and waits for readers, so callers run it when idle.
+func (s *Store) Checkpoint(ctx context.Context) error {
+	_, err := s.w.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`)
+	return err
+}
+
+// KV is one key/value pair.
+type KV struct {
+	K, V string
+}
+
+// Reader is the read surface shared by Store (committed state) and Tx (the
+// transaction's own view). Storage helpers take a Reader so they work both
+// inside and outside a write.
+type Reader interface {
+	Get(ctx context.Context, k string) (string, bool, error)
+	MGet(ctx context.Context, keys []string) (map[string]string, error)
+	Range(ctx context.Context, prefix, after string, limit int) ([]KV, error)
+	Count(ctx context.Context, prefix string) (int64, error)
+}
+
+type queryer interface {
+	QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error)
+	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
+}
+
+// PrefixEnd returns the smallest string greater than every string with the
+// given prefix, so [prefix, PrefixEnd(prefix)) is exactly the prefix's range.
+// An empty result means "unbounded".
+func PrefixEnd(prefix string) string {
+	b := []byte(prefix)
+	for i := len(b) - 1; i >= 0; i-- {
+		if b[i] < 0xff {
+			b[i]++
+			return string(b[:i+1])
+		}
+	}
+	return ""
+}
+
+func get(ctx context.Context, q queryer, k string) (string, bool, error) {
+	var v string
+	err := q.QueryRowContext(ctx, `SELECT v FROM kv WHERE k = ?`, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+
+// mgetBatch stays well under SQLite's default 32766 bound-parameter cap.
+const mgetBatch = 500
+
+func mget(ctx context.Context, q queryer, keys []string) (map[string]string, error) {
+	out := make(map[string]string, len(keys))
+	for start := 0; start < len(keys); start += mgetBatch {
+		end := min(start+mgetBatch, len(keys))
+		chunk := keys[start:end]
+		args := make([]any, len(chunk))
+		for i, k := range chunk {
+			args[i] = k
+		}
+		rows, err := q.QueryContext(ctx, `SELECT k, v FROM kv WHERE k IN (?`+strings.Repeat(",?", len(chunk)-1)+`)`, args...)
+		if err != nil {
+			return nil, err
+		}
+		for rows.Next() {
+			var k, v string
+			if err := rows.Scan(&k, &v); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			out[k] = v
+		}
+		if err := rows.Close(); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}
+
+// rangeQuery returns keys with prefix, strictly after `after` (when non-empty),
+// in key order, at most limit (<= 0 means all).
+func rangeQuery(ctx context.Context, q queryer, prefix, after string, limit int) ([]KV, error) {
+	lo, loOp := prefix, ">="
+	if after != "" && after >= prefix {
+		lo, loOp = after, ">"
+	}
+	query := `SELECT k, v FROM kv WHERE k ` + loOp + ` ?`
+	args := []any{lo}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	query += ` ORDER BY k`
+	if limit > 0 {
+		query += ` LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := q.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		out = append(out, kv)
+	}
+	return out, rows.Err()
+}
+
+func count(ctx context.Context, q queryer, prefix string) (int64, error) {
+	query := `SELECT count(*) FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end := PrefixEnd(prefix); end != "" {
+		query += ` AND k < ?`
+		args = append(args, end)
+	}
+	var n int64
+	err := q.QueryRowContext(ctx, query, args...).Scan(&n)
+	return n, err
+}
+
+// Get returns the committed value of k.
+func (s *Store) Get(ctx context.Context, k string) (string, bool, error) { return get(ctx, s.r, k) }
+
+// MGet returns the committed values of the keys that exist.
+func (s *Store) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, s.r, keys)
+}
+
+// Range returns committed keys under prefix after `after`, in key order.
+func (s *Store) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, s.r, prefix, after, limit)
+}
+
+// Count returns how many committed keys sit under prefix.
+func (s *Store) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, s.r, prefix)
+}
+
+// ScanPrefix walks every committed key under prefix in pages of `page`, each
+// page its own short read so a long walk never pins a WAL snapshot (which
+// would stop checkpoints and let the WAL grow without bound).
+func (s *Store) ScanPrefix(ctx context.Context, prefix string, page int, fn func([]KV) error) error {
+	if page <= 0 {
+		page = 5000
+	}
+	after := ""
+	for {
+		batch, err := rangeQuery(ctx, s.r, prefix, after, page)
+		if err != nil {
+			return err
+		}
+		if len(batch) == 0 {
+			return nil
+		}
+		if err := fn(batch); err != nil {
+			return err
+		}
+		if len(batch) < page {
+			return nil
+		}
+		after = batch[len(batch)-1].K
+	}
+}
+
+// CountAll counts every row. A full scan — callers cache it.
+func (s *Store) CountAll(ctx context.Context) (int64, error) {
+	var n int64
+	err := s.r.QueryRowContext(ctx, `SELECT count(*) FROM kv`).Scan(&n)
+	return n, err
+}
+
+// ErrClosed is returned by Update on a closed store.
+var ErrClosed = errors.New("kv: store closed")
+
+// Update runs fn atomically: its writes (and the change events they publish)
+// commit together or not at all. fn may share a SQLite transaction with other
+// concurrent Updates (group commit) but is isolated from their failures by a
+// savepoint. Never call Update from inside fn.
+//
+// ctx bounds only the wait for the writer. Once fn starts it runs to
+// completion: interrupting a statement inside a shared transaction would roll
+// back every request in the group.
+func (s *Store) Update(ctx context.Context, fn func(tx *Tx) error) error {
+	req := &writeReq{fn: fn, done: make(chan error, 1)}
+	select {
+	case s.writes <- req:
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-s.stopped:
+		return ErrClosed
+	}
+	return <-req.done
+}
+
+// writer is the single goroutine that executes writes.
+func (s *Store) writer() {
+	for {
+		var first *writeReq
+		select {
+		case first = <-s.writes:
+		case <-s.stopped:
+			return
+		}
+		group := []*writeReq{first}
+	drain:
+		for len(group) < maxGroup {
+			select {
+			case r := <-s.writes:
+				group = append(group, r)
+			default:
+				break drain
+			}
+		}
+		s.writeMu.Lock()
+		s.runGroup(group)
+		s.writeMu.Unlock()
+	}
+}
+
+// group is the state shared by the requests of one commit.
+type group struct {
+	tx      *sql.Tx
+	pending map[string]*streamPending
+	stmts   map[*sql.Stmt]*sql.Stmt
+}
+
+func (g *group) snapshot() map[string]streamPending {
+	out := make(map[string]streamPending, len(g.pending))
+	for k, p := range g.pending {
+		out[k] = *p
+	}
+	return out
+}
+
+func (g *group) restore(snap map[string]streamPending) {
+	g.pending = make(map[string]*streamPending, len(snap))
+	for k, p := range snap {
+		p := p
+		g.pending[k] = &p
+	}
+}
+
+func (s *Store) runGroup(reqs []*writeReq) {
+	ctx := context.Background()
+	results := make([]error, len(reqs))
+	fail := func(err error) {
+		for i, r := range reqs {
+			if results[i] == nil {
+				results[i] = err
+			}
+			r.done <- results[i]
+		}
+	}
+
+	sqlTx, err := s.w.BeginTx(ctx, nil)
+	if err != nil {
+		fail(fmt.Errorf("begin: %w", err))
+		return
+	}
+	g := &group{tx: sqlTx}
+	for i, r := range reqs {
+		if _, err := sqlTx.ExecContext(ctx, `SAVEPOINT req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("savepoint: %w", err))
+			return
+		}
+		snap := g.snapshot()
+		tx := &Tx{ctx: ctx, s: s, g: g}
+		err := r.fn(tx)
+		if err == nil {
+			err = tx.flushEvents()
+		}
+		if err != nil {
+			results[i] = err
+			if _, rbErr := sqlTx.ExecContext(ctx, `ROLLBACK TO req`); rbErr != nil {
+				_ = sqlTx.Rollback()
+				fail(fmt.Errorf("rollback to savepoint: %w (after %v)", rbErr, err))
+				return
+			}
+			g.restore(snap)
+		}
+		if _, err := sqlTx.ExecContext(ctx, `RELEASE req`); err != nil {
+			_ = sqlTx.Rollback()
+			fail(fmt.Errorf("release savepoint: %w", err))
+			return
+		}
+	}
+	if err := (&Tx{ctx: ctx, s: s, g: g}).flushHWM(); err != nil {
+		_ = sqlTx.Rollback()
+		fail(err)
+		return
+	}
+	if err := sqlTx.Commit(); err != nil {
+		fail(fmt.Errorf("commit: %w", err))
+		return
+	}
+	s.afterCommit(g.pending)
+	for i, r := range reqs {
+		r.done <- results[i]
+	}
+}
+
+// Tx is one Update's view of the write transaction. Its reads see its own
+// writes (and those of earlier requests in the same group).
+type Tx struct {
+	ctx    context.Context
+	s      *Store
+	g      *group
+	events []map[string]string
+}
+
+func (t *Tx) sqlTx() *sql.Tx { return t.g.tx }
+
+// stmt binds one of the store's prepared statements to the transaction.
+func (t *Tx) stmt(st *sql.Stmt) *sql.Stmt {
+	if t.g.stmts == nil {
+		t.g.stmts = make(map[*sql.Stmt]*sql.Stmt, 5)
+	}
+	bound := t.g.stmts[st]
+	if bound == nil {
+		bound = t.g.tx.StmtContext(t.ctx, st)
+		t.g.stmts[st] = bound
+	}
+	return bound
+}
+
+// Context is the context the transaction runs under.
+func (t *Tx) Context() context.Context { return t.ctx }
+
+func (t *Tx) Get(ctx context.Context, k string) (string, bool, error) {
+	var v string
+	err := t.stmt(t.s.stGet).QueryRowContext(ctx, k).Scan(&v)
+	if errors.Is(err, sql.ErrNoRows) {
+		return "", false, nil
+	}
+	if err != nil {
+		return "", false, err
+	}
+	return v, true, nil
+}
+func (t *Tx) MGet(ctx context.Context, keys []string) (map[string]string, error) {
+	return mget(ctx, t.sqlTx(), keys)
+}
+func (t *Tx) Range(ctx context.Context, prefix, after string, limit int) ([]KV, error) {
+	return rangeQuery(ctx, t.sqlTx(), prefix, after, limit)
+}
+func (t *Tx) Count(ctx context.Context, prefix string) (int64, error) {
+	return count(ctx, t.sqlTx(), prefix)
+}
+
+// Exec runs a raw statement in the transaction — for the caller's derived
+// tables (FTS), never for kv rows (which would skip events).
+func (t *Tx) Exec(query string, args ...any) (sql.Result, error) {
+	return t.sqlTx().ExecContext(t.ctx, query, args...)
+}
+
+// QueryRow runs a raw single-row query in the transaction.
+func (t *Tx) QueryRow(query string, args ...any) *sql.Row {
+	return t.sqlTx().QueryRowContext(t.ctx, query, args...)
+}
+
+// Put sets k = v. Publishes `set` even when the value is unchanged, as Redis
+// keyspace notifications did.
+func (t *Tx) Put(k, v string) error {
+	if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, k, v); err != nil {
+		return fmt.Errorf("put %q: %w", k, err)
+	}
+	t.event("set", k)
+	return nil
+}
+
+// PutIfAbsent sets k = v only when k does not exist. Reports whether it wrote.
+// Publishes nothing when the key already existed (nothing changed).
+func (t *Tx) PutIfAbsent(k, v string) (bool, error) {
+	res, err := t.stmt(t.s.stPutIfAbsent).ExecContext(t.ctx, k, v)
+	if err != nil {
+		return false, fmt.Errorf("put %q: %w", k, err)
+	}
+	n, _ := res.RowsAffected()
+	if n > 0 {
+		t.event("set", k)
+	}
+	return n > 0, nil
+}
+
+// Delete removes k. Reports whether it existed; publishes `del` only then,
+// as Redis did.
+func (t *Tx) Delete(k string) (bool, error) {
+	res, err := t.stmt(t.s.stDelete).ExecContext(t.ctx, k)
+	if err != nil {
+		return false, fmt.Errorf("delete %q: %w", k, err)
+	}
+	n, _ := res.RowsAffected()
+	if n > 0 {
+		t.event("del", k)
+	}
+	return n > 0, nil
+}
+
+// DeletePrefix removes up to limit keys under prefix (limit <= 0: all) and
+// returns how many went. Bulk callers loop over separate Updates with a limit
+// so one transaction never grows to millions of rows.
+func (t *Tx) DeletePrefix(prefix string, limit int) (int64, error) {
+	end := PrefixEnd(prefix)
+	sel := `SELECT k FROM kv WHERE k >= ?`
+	args := []any{prefix}
+	if end != "" {
+		sel += ` AND k < ?`
+		args = append(args, end)
+	}
+	if limit > 0 {
+		sel += ` ORDER BY k LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := t.sqlTx().QueryContext(t.ctx, `DELETE FROM kv WHERE k IN (`+sel+`) RETURNING k`, args...)
+	if err != nil {
+		return 0, fmt.Errorf("delete prefix %q: %w", prefix, err)
+	}
+	var keys []string
+	for rows.Next() {
+		var k string
+		if err := rows.Scan(&k); err != nil {
+			rows.Close()
+			return 0, err
+		}
+		keys = append(keys, k)
+	}
+	if err := rows.Close(); err != nil {
+		return 0, err
+	}
+	for _, k := range keys {
+		t.event("del", k)
+	}
+	return int64(len(keys)), nil
+}
+
+func (t *Tx) event(op, k string) {
+	if t.s.opts.EventStream == "" || t.s.opts.EventKey == nil || !t.s.opts.EventKey(k) {
+		return
+	}
+	t.events = append(t.events, map[string]string{
+		"type": op,
+		"key":  k,
+		"ts":   fmt.Sprintf("%d", time.Now().UnixMilli()),
+	})
+}
+
+// flushEvents appends the queued change events to the event stream, inside
+// the transaction, right before commit.
+func (t *Tx) flushEvents() error {
+	if len(t.events) == 0 {
+		return nil
+	}
+	evs := t.events
+	t.events = nil
+	for _, e := range evs {
+		if _, err := t.Append(t.s.opts.EventStream, t.s.opts.EventMaxLen, e); err != nil {
+			return err
+		}
+	}
+	return nil
+}

internal/kv/kv_test.go

@@ -0,0 +1,314 @@
+package kv
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func openTest(t *testing.T, path string) *Store {
+	t.Helper()
+	if path == "" {
+		path = filepath.Join(t.TempDir(), "kv.db")
+	}
+	s, err := Open(path, Options{
+		EventStream: "meta:events",
+		EventMaxLen: 100_000,
+		EventKey:    func(k string) bool { return strings.HasPrefix(k, "file:") },
+	})
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+	return s
+}
+
+func update(t *testing.T, s *Store, fn func(tx *Tx) error) {
+	t.Helper()
+	if err := s.Update(context.Background(), fn); err != nil {
+		t.Fatalf("update: %v", err)
+	}
+}
+
+func events(t *testing.T, s *Store) []Entry {
+	t.Helper()
+	es, err := s.ReadAfter(context.Background(), "meta:events", "0", 0)
+	if err != nil {
+		t.Fatalf("read events: %v", err)
+	}
+	return es
+}
+
+// The replacement for keyspace notifications: a change and its event commit
+// together, with Redis' semantics — `set` even for an unchanged value, `del`
+// only for a key that existed, and nothing for keys outside the event scope.
+func TestChangesPublishEventsWithRedisSemantics(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		_ = tx.Put("file:A/title", "x") // unchanged value still publishes
+		_ = tx.Put("cid:bafk", "A")     // outside the event scope
+		_, _ = tx.Delete("file:A/missing")
+		_, _ = tx.Delete("file:A/title")
+		return nil
+	})
+	got := events(t, s)
+	want := []string{"set file:A/title", "set file:A/title", "del file:A/title"}
+	if len(got) != len(want) {
+		t.Fatalf("events = %+v, want %v", got, want)
+	}
+	for i, e := range got {
+		if e.Values["type"]+" "+e.Values["key"] != want[i] {
+			t.Errorf("event %d = %v, want %q", i, e.Values, want[i])
+		}
+	}
+}
+
+func TestRolledBackChangesPublishNothing(t *testing.T) {
+	s := openTest(t, "")
+	err := s.Update(context.Background(), func(tx *Tx) error {
+		_ = tx.Put("file:A/title", "x")
+		return errors.New("boom")
+	})
+	if err == nil {
+		t.Fatal("want the error back")
+	}
+	if _, ok, _ := s.Get(context.Background(), "file:A/title"); ok {
+		t.Fatal("rolled-back write is visible")
+	}
+	if es := events(t, s); len(es) != 0 {
+		t.Fatalf("rolled-back write published %+v", es)
+	}
+}
+
+func TestDeletePrefixPublishesOneDelPerKey(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b", "file:AB/c"} {
+			_ = tx.Put(k, "1")
+		}
+		return nil
+	})
+	var n int64
+	update(t, s, func(tx *Tx) error {
+		var err error
+		n, err = tx.DeletePrefix("file:A/", 0)
+		return err
+	})
+	if n != 2 {
+		t.Fatalf("deleted %d, want 2 (file:AB/ is not under file:A/)", n)
+	}
+	es := events(t, s)
+	if dels := es[3:]; len(dels) != 2 || dels[0].Values["type"] != "del" || dels[1].Values["type"] != "del" {
+		t.Fatalf("want two del events, got %+v", es)
+	}
+}
+
+// IDs are Redis-shaped and strictly increasing, within one millisecond and
+// across a reopen — the SSE cursors consumers persist depend on both.
+func TestStreamIDsAreMonotonicAcrossReopen(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "kv.db")
+	s, err := Open(path, Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ids []string
+	if err := s.Update(context.Background(), func(tx *Tx) error {
+		for i := 0; i < 50; i++ {
+			id, err := tx.Append("file:events", 0, map[string]string{"n": "x"})
+			if err != nil {
+				return err
+			}
+			ids = append(ids, id)
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	_ = s.Close()
+
+	s2 := openTest(t, path)
+	var after string
+	update(t, s2, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"n": "y"})
+		return err
+	})
+	ids = append(ids, after)
+	for i := 1; i < len(ids); i++ {
+		if !idLess(t, ids[i-1], ids[i]) {
+			t.Fatalf("id %d (%s) does not follow %s", i, ids[i], ids[i-1])
+		}
+	}
+	if !strings.Contains(ids[0], "-") || strings.HasPrefix(ids[0], "0") {
+		t.Fatalf("id %q is not the unpadded <ms>-<seq> form", ids[0])
+	}
+}
+
+func idLess(t *testing.T, a, b string) bool {
+	t.Helper()
+	am, as, err := ParseID(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bm, bs, err := ParseID(b)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return am < bm || (am == bm && as < bs)
+}
+
+// The watcher's reset clears file:events. The next ID must still exceed every
+// ID issued before the clear, or a consumer holding an old cursor would skip
+// the reset event.
+func TestClearStreamKeepsTheHighWaterMark(t *testing.T) {
+	s := openTest(t, "")
+	var before, after string
+	update(t, s, func(tx *Tx) error {
+		var err error
+		for i := 0; i < 5; i++ {
+			if before, err = tx.Append("file:events", 0, map[string]string{"type": "add"}); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	update(t, s, func(tx *Tx) error { return tx.ClearStream("file:events") })
+	if n, _ := s.Len(context.Background(), "file:events"); n != 0 {
+		t.Fatalf("len after clear = %d", n)
+	}
+	update(t, s, func(tx *Tx) error {
+		var err error
+		after, err = tx.Append("file:events", 0, map[string]string{"type": "reset"})
+		return err
+	})
+	if !idLess(t, before, after) {
+		t.Fatalf("post-clear id %s does not follow pre-clear %s", after, before)
+	}
+	got, err := s.ReadAfter(context.Background(), "file:events", before, 10)
+	if err != nil || len(got) != 1 || got[0].Values["type"] != "reset" {
+		t.Fatalf("an old cursor must see the reset: %+v err=%v", got, err)
+	}
+}
+
+// MAXLEN ~ semantics across many small transactions (the common case: one
+// event per write).
+func TestStreamTrimsAcrossTransactions(t *testing.T) {
+	defer func(old int64) { trimEvery = old }(trimEvery)
+	trimEvery = 100
+	s := openTest(t, "")
+	const maxLen = 50
+	for i := int64(0); i < 3*trimEvery; i++ {
+		update(t, s, func(tx *Tx) error {
+			_, err := tx.Append("x", maxLen, map[string]string{"i": "1"})
+			return err
+		})
+	}
+	n, err := s.Len(context.Background(), "x")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n > maxLen+trimEvery {
+		t.Fatalf("stream holds %d entries, want <= %d", n, maxLen+trimEvery)
+	}
+	oldest, ok, err := s.Oldest(context.Background(), "x")
+	if err != nil || !ok {
+		t.Fatal(err)
+	}
+	if last, _ := s.LastID(context.Background(), "x"); !idLess(t, oldest.ID, last) {
+		t.Fatalf("oldest %s should precede last %s", oldest.ID, last)
+	}
+}
+
+// A subscriber that took the wake channel before reading is woken by the next
+// commit — the property the SSE pump relies on to not miss an append.
+func TestWaitIsClosedByTheNextCommit(t *testing.T) {
+	s := openTest(t, "")
+	wake := s.Wait("meta:events")
+	update(t, s, func(tx *Tx) error { return tx.Put("file:A/x", "1") })
+	select {
+	case <-wake:
+	case <-time.After(2 * time.Second):
+		t.Fatal("commit did not wake the waiter")
+	}
+}
+
+func TestReadAfterZeroFormsReadFromTheStart(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		_, err := tx.Append("x", 0, map[string]string{"n": "1"})
+		return err
+	})
+	for _, id := range []string{"", "0", "0-0"} {
+		got, err := s.ReadAfter(context.Background(), "x", id, 10)
+		if err != nil || len(got) != 1 {
+			t.Errorf("ReadAfter(%q) = %d entries, err %v; want 1", id, len(got), err)
+		}
+	}
+}
+
+func TestRangeIsExactlyThePrefix(t *testing.T) {
+	s := openTest(t, "")
+	update(t, s, func(tx *Tx) error {
+		for _, k := range []string{"file:A/a", "file:A/b/c", "file:A0", "file:AB/x", "file:A"} {
+			_ = tx.Put(k, "v")
+		}
+		return nil
+	})
+	rows, err := s.Range(context.Background(), "file:A/", "", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 || rows[0].K != "file:A/a" || rows[1].K != "file:A/b/c" {
+		t.Fatalf("range = %+v", rows)
+	}
+}
+
+// Concurrent Updates share commits; a failing one must roll back alone and
+// leave its neighbours' writes and events intact.
+func TestGroupCommitIsolatesFailures(t *testing.T) {
+	s := openTest(t, "")
+	const n = 200
+	errs := make(chan error, n)
+	for i := 0; i < n; i++ {
+		go func(i int) {
+			errs <- s.Update(context.Background(), func(tx *Tx) error {
+				k := fmt.Sprintf("file:R%03d/title", i)
+				if err := tx.Put(k, "x"); err != nil {
+					return err
+				}
+				if i%10 == 0 {
+					return errors.New("fail")
+				}
+				return nil
+			})
+		}(i)
+	}
+	failed := 0
+	for i := 0; i < n; i++ {
+		if <-errs != nil {
+			failed++
+		}
+	}
+	if failed != n/10 {
+		t.Fatalf("failed = %d, want %d", failed, n/10)
+	}
+	rows, _ := s.Range(context.Background(), "file:", "", 0)
+	if len(rows) != n-n/10 {
+		t.Fatalf("committed %d rows, want %d", len(rows), n-n/10)
+	}
+	if es := events(t, s); len(es) != n-n/10 {
+		t.Fatalf("events = %d, want %d (one per committed write, none for failures)", len(es), n-n/10)
+	}
+	for _, r := range rows {
+		var i int
+		fmt.Sscanf(r.K, "file:R%03d/title", &i)
+		if i%10 == 0 {
+			t.Fatalf("failed request's write %s committed", r.K)
+		}
+	}
+}

internal/kv/stream.go

@@ -0,0 +1,356 @@
+package kv
+
+// Append-only event streams, as rows of the kv table.
+//
+// A stream entry is the row
+//
+//	ev:<stream>/<ms:13>-<seq:10>  →  JSON object of the entry's string fields
+//
+// Zero padding makes key order equal ID order, so reading "after an ID" is a
+// plain range read and trimming is a range delete. On the wire IDs are the
+// unpadded Redis form (`1700000000000-0`): meta-sort, meta-share and
+// meta-stremio persist SSE cursors to disk across the Redis → SQLite switch,
+// and those compare as <ms>-<seq> pairs.
+//
+// The last issued ID is kept in `evhwm:<stream>` and updated in the same
+// transaction as the append. That high-water mark, not the newest surviving
+// row, seeds the next ID — so Clear (the watcher's `reset`) never reissues an
+// ID an old cursor would treat as already seen.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Entry is one stream entry.
+type Entry struct {
+	ID     string
+	Values map[string]string
+}
+
+type streamID struct{ ms, seq int64 }
+
+func (id streamID) String() string {
+	return strconv.FormatInt(id.ms, 10) + "-" + strconv.FormatInt(id.seq, 10)
+}
+
+func (id streamID) padded() string { return fmt.Sprintf("%013d-%010d", id.ms, id.seq) }
+
+func (id streamID) less(o streamID) bool {
+	return id.ms < o.ms || (id.ms == o.ms && id.seq < o.seq)
+}
+
+// ParseID parses a Redis-style stream ID. "", "0", "0-0", "-" all parse as the
+// zero ID (before everything). A bare "<ms>" means "<ms>-0".
+func ParseID(s string) (ms, seq int64, err error) {
+	if s == "" || s == "-" || s == "0" {
+		return 0, 0, nil
+	}
+	msPart, seqPart, hasSeq := strings.Cut(s, "-")
+	ms, err = strconv.ParseInt(msPart, 10, 64)
+	if err != nil || ms < 0 {
+		return 0, 0, fmt.Errorf("invalid stream id %q", s)
+	}
+	if hasSeq {
+		seq, err = strconv.ParseInt(seqPart, 10, 64)
+		if err != nil || seq < 0 {
+			return 0, 0, fmt.Errorf("invalid stream id %q", s)
+		}
+	}
+	return ms, seq, nil
+}
+
+func streamPrefix(stream string) string { return "ev:" + stream + "/" }
+func hwmKey(stream string) string       { return "evhwm:" + stream }
+
+// streamState is the in-memory view of a stream's high-water mark plus the
+// broadcast channel subscribers wait on. Updated only after a commit.
+type streamState struct {
+	loaded    bool
+	hwm       streamID
+	sinceTrim int64 // committed appends since the last trim
+	notify    chan struct{}
+}
+
+// streamPending is what a transaction did to one stream, applied to
+// streamState on commit (and discarded on rollback).
+type streamPending struct {
+	hwm        streamID
+	appends    int64
+	trimmed    bool
+	lastTrimAt int64 // value of appends when this transaction last trimmed
+}
+
+func (s *Store) state(stream string) *streamState {
+	st := s.streams[stream]
+	if st == nil {
+		st = &streamState{notify: make(chan struct{})}
+		s.streams[stream] = st
+	}
+	return st
+}
+
+// Wait returns a channel closed by the next commit that appends to stream.
+// Take it BEFORE reading, so an append between the read and the wait is not
+// missed.
+func (s *Store) Wait(stream string) <-chan struct{} {
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	return s.state(stream).notify
+}
+
+func (s *Store) afterCommit(pending map[string]*streamPending) {
+	if len(pending) == 0 {
+		return
+	}
+	s.streamMu.Lock()
+	defer s.streamMu.Unlock()
+	for name, p := range pending {
+		st := s.state(name)
+		st.loaded = true
+		st.hwm = p.hwm
+		if p.trimmed {
+			st.sinceTrim = p.appends - p.lastTrimAt
+		} else {
+			st.sinceTrim += p.appends
+		}
+		close(st.notify)
+		st.notify = make(chan struct{})
+	}
+}
+
+// hwm returns the stream's last issued ID as seen by this transaction.
+func (t *Tx) hwm(stream string) (streamID, error) {
+	if p := t.g.pending[stream]; p != nil {
+		return p.hwm, nil
+	}
+	t.s.streamMu.Lock()
+	st := t.s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	t.s.streamMu.Unlock()
+	if loaded {
+		return hwm, nil
+	}
+	v, ok, err := t.Get(t.ctx, hwmKey(stream))
+	if err != nil {
+		return streamID{}, err
+	}
+	if !ok {
+		return streamID{}, nil
+	}
+	ms, seq, err := ParseID(v)
+	if err != nil {
+		return streamID{}, err
+	}
+	return streamID{ms, seq}, nil
+}
+
+// Append adds an entry to stream and returns its ID. The stream is trimmed to
+// about maxLen entries (maxLen <= 0: unbounded).
+func (t *Tx) Append(stream string, maxLen int64, fields map[string]string) (string, error) {
+	last, err := t.hwm(stream)
+	if err != nil {
+		return "", fmt.Errorf("stream %s hwm: %w", stream, err)
+	}
+	id := streamID{ms: time.Now().UnixMilli()}
+	if !last.less(id) {
+		id = streamID{ms: last.ms, seq: last.seq + 1}
+	}
+	body, err := json.Marshal(fields)
+	if err != nil {
+		return "", err
+	}
+	if _, err := t.stmt(t.s.stInsert).ExecContext(t.ctx, streamPrefix(stream)+id.padded(), string(body)); err != nil {
+		return "", fmt.Errorf("append %s: %w", stream, err)
+	}
+	if t.g.pending == nil {
+		t.g.pending = make(map[string]*streamPending)
+	}
+	p := t.g.pending[stream]
+	if p == nil {
+		p = &streamPending{}
+		t.g.pending[stream] = p
+	}
+	p.hwm = id
+	p.appends++
+	// Trim in steps, not per append: MAXLEN ~ semantics, like the XADD it
+	// replaces. The count spans transactions (most append one event each).
+	if maxLen > 0 {
+		t.s.streamMu.Lock()
+		since := t.s.state(stream).sinceTrim
+		t.s.streamMu.Unlock()
+		if !p.trimmed {
+			since += p.appends
+		} else {
+			since = p.appends - p.lastTrimAt
+		}
+		if since >= trimEvery {
+			if err := t.trim(stream, maxLen); err != nil {
+				return "", err
+			}
+			p.trimmed, p.lastTrimAt = true, p.appends
+		}
+	}
+	return id.String(), nil
+}
+
+// trimEvery is how many appends pass between trims. A trim locates its cut
+// point by walking maxLen entries back from the newest, so it is kept rare;
+// the cost is overshooting maxLen by at most trimEvery entries.
+var trimEvery int64 = 10_000 // var so tests can lower it
+
+func (t *Tx) trim(stream string, maxLen int64) error {
+	prefix := streamPrefix(stream)
+	var cut string
+	err := t.sqlTx().QueryRowContext(t.ctx,
+		`SELECT k FROM kv WHERE k >= ? AND k < ? ORDER BY k DESC LIMIT 1 OFFSET ?`,
+		prefix, PrefixEnd(prefix), maxLen).Scan(&cut)
+	if err != nil {
+		return nil // fewer than maxLen entries (sql.ErrNoRows): nothing to trim
+	}
+	_, err = t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k <= ?`, prefix, cut)
+	return err
+}
+
+// flushHWM persists each touched stream's high-water mark — once per
+// transaction, not per append.
+func (t *Tx) flushHWM() error {
+	for stream, p := range t.g.pending {
+		if _, err := t.stmt(t.s.stPut).ExecContext(t.ctx, hwmKey(stream), p.hwm.String()); err != nil {
+			return fmt.Errorf("stream %s hwm: %w", stream, err)
+		}
+	}
+	return nil
+}
+
+// TrimAll trims stream to maxLen inside the transaction regardless of how many
+// appends this transaction made. Used after an append burst split over many
+// short transactions.
+func (t *Tx) TrimAll(stream string, maxLen int64) error { return t.trim(stream, maxLen) }
+
+// ClearStream deletes every entry of stream, keeping its high-water mark.
+func (t *Tx) ClearStream(stream string) error {
+	prefix := streamPrefix(stream)
+	_, err := t.sqlTx().ExecContext(t.ctx, `DELETE FROM kv WHERE k >= ? AND k < ?`, prefix, PrefixEnd(prefix))
+	return err
+}
+
+func decodeEntry(stream, k, v string) (Entry, error) {
+	padded := strings.TrimPrefix(k, streamPrefix(stream))
+	ms, seq, err := ParseID(padded)
+	if err != nil {
+		return Entry{}, err
+	}
+	var values map[string]string
+	if err := json.Unmarshal([]byte(v), &values); err != nil {
+		return Entry{}, fmt.Errorf("decode %s: %w", k, err)
+	}
+	return Entry{ID: streamID{ms, seq}.String(), Values: values}, nil
+}
+
+func decodeEntries(stream string, rows []KV) ([]Entry, error) {
+	out := make([]Entry, 0, len(rows))
+	for _, kv := range rows {
+		e, err := decodeEntry(stream, kv.K, kv.V)
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, e)
+	}
+	return out, nil
+}
+
+// ReadAfter returns up to n entries strictly after id, oldest first. "0",
+// "0-0" and "" read from the beginning.
+func (s *Store) ReadAfter(ctx context.Context, stream, id string, n int) ([]Entry, error) {
+	ms, seq, err := ParseID(id)
+	if err != nil {
+		return nil, err
+	}
+	prefix := streamPrefix(stream)
+	after := ""
+	if ms != 0 || seq != 0 {
+		after = prefix + streamID{ms, seq}.padded()
+	}
+	rows, err := rangeQuery(ctx, s.r, prefix, after, n)
+	if err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, rows)
+}
+
+// Oldest returns the oldest surviving entry, or ok=false for an empty stream.
+func (s *Store) Oldest(ctx context.Context, stream string) (Entry, bool, error) {
+	rows, err := rangeQuery(ctx, s.r, streamPrefix(stream), "", 1)
+	if err != nil || len(rows) == 0 {
+		return Entry{}, false, err
+	}
+	e, err := decodeEntry(stream, rows[0].K, rows[0].V)
+	return e, err == nil, err
+}
+
+// LastID returns the last ID ever issued on stream ("0-0" if none) — the
+// cursor meaning "only entries appended from now on".
+func (s *Store) LastID(ctx context.Context, stream string) (string, error) {
+	s.streamMu.Lock()
+	st := s.state(stream)
+	loaded, hwm := st.loaded, st.hwm
+	s.streamMu.Unlock()
+	if loaded {
+		return hwm.String(), nil
+	}
+	v, ok, err := s.Get(ctx, hwmKey(stream))
+	if err != nil {
+		return "", err
+	}
+	if !ok {
+		return "0-0", nil
+	}
+	return v, nil
+}
+
+// Len is the number of surviving entries.
+func (s *Store) Len(ctx context.Context, stream string) (int64, error) {
+	return count(ctx, s.r, streamPrefix(stream))
+}
+
+// RevRange returns up to n entries newest first, stopping at (excluding)
+// entries at or before afterID when afterID is non-empty.
+func (s *Store) RevRange(ctx context.Context, stream, afterID string, n int) ([]Entry, error) {
+	prefix := streamPrefix(stream)
+	query := `SELECT k, v FROM kv WHERE k > ? AND k < ? ORDER BY k DESC`
+	lo := prefix
+	if afterID != "" {
+		ms, seq, err := ParseID(afterID)
+		if err != nil {
+			return nil, err
+		}
+		lo = prefix + streamID{ms, seq}.padded()
+	}
+	args := []any{lo, PrefixEnd(prefix)}
+	if n > 0 {
+		query += ` LIMIT ?`
+		args = append(args, n)
+	}
+	rows, err := s.r.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var kvs []KV
+	for rows.Next() {
+		var kv KV
+		if err := rows.Scan(&kv.K, &kv.V); err != nil {
+			return nil, err
+		}
+		kvs = append(kvs, kv)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return decodeEntries(stream, kvs)
+}

internal/meshdisco/node.go

@@ -0,0 +1,439 @@
+package meshdisco
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"sort"
+	"sync"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/ipv4"
+	"golang.org/x/sys/unix"
+)
+
+// Config configures a Node. Payload is a function rather than a value so the
+// advertise is rebuilt from live state on every tick — that is what keeps the
+// advertised core URLs and GET /urls from drifting apart.
+type Config struct {
+	Name     string
+	Instance string
+	Version  string
+
+	Group    string
+	Port     int
+	Interval time.Duration
+
+	// Payload returns this node's status ("starting" / "running") and its
+	// resources. The envelope and NodeInfo are filled in by the Node.
+	Payload func() (status string, resources []Resource)
+}
+
+func (c *Config) applyDefaults() {
+	if c.Group == "" {
+		c.Group = DefaultGroup
+	}
+	if c.Port == 0 {
+		c.Port = DefaultPort
+	}
+	if c.Interval <= 0 {
+		c.Interval = DefaultInterval
+	}
+}
+
+type seen struct {
+	node      NodeInfo
+	resources []Resource
+	addr      string
+	lastSeen  time.Time
+}
+
+// Node is one beacon v2 participant: it advertises its resources, answers
+// probes, and keeps a TTL map of every other node it has heard from.
+type Node struct {
+	cfg   Config
+	group net.IP
+
+	pc     *ipv4.PacketConn
+	closer interface{ Close() error }
+
+	// ifaces are the multicast-capable interfaces we joined and send on.
+	// A container attached to two docker networks (metashare-app on `pcs`
+	// and `metamesh-mesh`, metawatch-* on `metawatch` and `metamesh-mesh`)
+	// has two entries here, and MUST be advertised on both — sending via the
+	// default route alone silently covers only one network.
+	ifaces []net.Interface
+
+	// sendMu serialises SetMulticastInterface + WriteTo, which are a
+	// two-step operation on one shared socket.
+	sendMu sync.Mutex
+
+	mu    sync.RWMutex
+	nodes map[string]*seen
+
+	stopChan chan struct{}
+	stopOnce sync.Once
+	wg       sync.WaitGroup
+}
+
+// New creates a Node. It does not touch the network until Start.
+func New(cfg Config) *Node {
+	cfg.applyDefaults()
+	return &Node{
+		cfg:      cfg,
+		nodes:    make(map[string]*seen),
+		stopChan: make(chan struct{}),
+	}
+}
+
+// Start opens the multicast socket, joins the group on every eligible
+// interface, and begins advertising. A join failure on one interface is logged
+// and skipped rather than fatal — a host that blocks multicast degrades
+// instead of failing to boot.
+func (n *Node) Start() error {
+	n.group = net.ParseIP(n.cfg.Group)
+	if n.group == nil || n.group.To4() == nil {
+		return fmt.Errorf("beacon: invalid multicast group %q", n.cfg.Group)
+	}
+
+	lc := net.ListenConfig{Control: reusePort}
+	conn, err := lc.ListenPacket(context.Background(), "udp4", fmt.Sprintf(":%d", n.cfg.Port))
+	if err != nil {
+		return fmt.Errorf("beacon: listen on :%d: %w", n.cfg.Port, err)
+	}
+	n.closer = conn
+
+	pc := ipv4.NewPacketConn(conn)
+	// TTL 1: link-local only. Discovery never routes off the local segment;
+	// the two-host topology keeps two separate meshes by design.
+	if err := pc.SetMulticastTTL(1); err != nil {
+		log.Printf("[beacon] Warning: could not set multicast TTL: %v", err)
+	}
+	n.pc = pc
+
+	n.ifaces = eligibleInterfaces()
+	if len(n.ifaces) == 0 {
+		log.Println("[beacon] Warning: no multicast-capable interfaces found; discovery will be inert")
+	}
+
+	group := &net.UDPAddr{IP: n.group}
+	joined := 0
+	for i := range n.ifaces {
+		if err := pc.JoinGroup(&n.ifaces[i], group); err != nil {
+			log.Printf("[beacon] Warning: join %s on %s failed: %v", n.cfg.Group, n.ifaces[i].Name, err)
+			continue
+		}
+		joined++
+	}
+	log.Printf("[beacon] Listening on %s:%d as %s/%s (joined %d of %d interfaces)",
+		n.cfg.Group, n.cfg.Port, n.cfg.Name, n.cfg.Instance, joined, len(n.ifaces))
+
+	n.wg.Add(2)
+	go n.readLoop()
+	go n.advertiseLoop()
+
+	return nil
+}
+
+// Stop multicasts a bye — so neighbours drop us immediately instead of
+// waiting out the staleness window — then closes the socket and waits for the
+// loops to exit.
+func (n *Node) Stop() error {
+	var err error
+	n.stopOnce.Do(func() {
+		n.broadcast(Message{Proto: Proto, V: Version, Type: TypeBye, Node: n.nodeInfo("")})
+		close(n.stopChan)
+		if n.closer != nil {
+			err = n.closer.Close()
+		}
+		n.wg.Wait()
+	})
+	return err
+}
+
+// Probe multicasts a probe on every interface. Every node owning a resource
+// matching one of want (every node, when want is empty) replies immediately,
+// so a caller does not have to wait out an interval.
+func (n *Node) Probe(want ...string) {
+	n.broadcast(Message{Proto: Proto, V: Version, Type: TypeProbe, From: n.cfg.Instance, Want: want})
+}
+
+// Neighbors returns every node heard from within the staleness window,
+// sorted by (name, instance). Expired entries are dropped as a side effect —
+// expiry is a read-time concern, so there is no reaper goroutine.
+func (n *Node) Neighbors() []Neighbor {
+	cutoff := time.Now().Add(-n.cfg.Interval * LivenessFactor)
+
+	n.mu.Lock()
+	out := make([]Neighbor, 0, len(n.nodes))
+	for k, s := range n.nodes {
+		if s.lastSeen.Before(cutoff) {
+			delete(n.nodes, k)
+			continue
+		}
+		out = append(out, newNeighbor(s.node, s.resources, s.addr, s.lastSeen))
+	}
+	n.mu.Unlock()
+
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Name != out[j].Name {
+			return out[i].Name < out[j].Name
+		}
+		return out[i].Instance < out[j].Instance
+	})
+	return out
+}
+
+// NeighborsByName collapses multiple instances of one name down to the most
+// recently seen — the shape the nav menu wants.
+func (n *Node) NeighborsByName() []Neighbor {
+	best := make(map[string]Neighbor)
+	for _, nb := range n.Neighbors() {
+		if cur, ok := best[nb.Name]; !ok || nb.LastSeen > cur.LastSeen {
+			best[nb.Name] = nb
+		}
+	}
+	out := make([]Neighbor, 0, len(best))
+	for _, nb := range best {
+		out = append(out, nb)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Self returns this node as a row, so callers can render themselves without
+// waiting to hear their own multicast echo back.
+func (n *Node) Self() Neighbor {
+	status, resources := n.payload()
+	return newNeighbor(*n.nodeInfo(status), resources, "", time.Now())
+}
+
+func (n *Node) payload() (string, []Resource) {
+	if n.cfg.Payload == nil {
+		return "running", nil
+	}
+	return n.cfg.Payload()
+}
+
+func (n *Node) nodeInfo(status string) *NodeInfo {
+	return &NodeInfo{Name: n.cfg.Name, Instance: n.cfg.Instance, Version: n.cfg.Version, Status: status}
+}
+
+func (n *Node) advertiseMsg() Message {
+	status, resources := n.payload()
+	return Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: n.nodeInfo(status), Resources: resources}
+}
+
+func (n *Node) readLoop() {
+	defer n.wg.Done()
+	buf := make([]byte, ReadBufferSize)
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		default:
+		}
+
+		nRead, _, src, err := n.pc.ReadFrom(buf)
+		if err != nil {
+			select {
+			case <-n.stopChan:
+				return
+			default:
+			}
+			log.Printf("[beacon] Read error: %v", err)
+			continue
+		}
+
+		msg, ok := Parse(buf[:nRead])
+		if !ok {
+			continue // v1 or foreign traffic on the shared group
+		}
+
+		switch msg.Type {
+		case TypeProbe:
+			if msg.From == n.cfg.Instance {
+				continue
+			}
+			ad := n.advertiseMsg()
+			wanted := len(msg.Want) == 0
+			for _, r := range ad.Resources {
+				if r.MatchesAny(msg.Want) {
+					wanted = true
+					break
+				}
+			}
+			if wanted {
+				// Unicast straight back to the prober's source address; its
+				// ephemeral source port is where it is listening.
+				n.send(ad, src)
+			}
+		case TypeAdvertise, TypeBye:
+			// Drop our own multicast echo. Loopback stays enabled so a
+			// single-container test still sees traffic; we filter by identity.
+			if msg.Node.Instance == n.cfg.Instance {
+				continue
+			}
+			n.record(msg, src)
+		}
+	}
+}
+
+func (n *Node) record(msg Message, src net.Addr) {
+	addr := ""
+	if src != nil {
+		if udp, ok := src.(*net.UDPAddr); ok {
+			addr = udp.IP.String()
+		} else {
+			addr = src.String()
+		}
+	}
+
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if msg.Type == TypeBye {
+		delete(n.nodes, msg.Node.Instance)
+		return
+	}
+	n.nodes[msg.Node.Instance] = &seen{node: *msg.Node, resources: msg.Resources, addr: addr, lastSeen: time.Now()}
+}
+
+func (n *Node) advertiseLoop() {
+	defer n.wg.Done()
+
+	// Advertise immediately, then probe once so the map fills without
+	// waiting a full interval for everyone else's ticker.
+	n.broadcast(n.advertiseMsg())
+	n.Probe()
+
+	ticker := time.NewTicker(n.cfg.Interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-n.stopChan:
+			return
+		case <-ticker.C:
+			n.broadcast(n.advertiseMsg())
+		}
+	}
+}
+
+func encode(msg Message) ([]byte, bool) {
+	body, err := json.Marshal(msg)
+	if err != nil {
+		return nil, false
+	}
+	if len(body) > MaxDatagram {
+		log.Printf("[beacon] Warning: %d-byte datagram exceeds the fragmentation-safe %d", len(body), MaxDatagram)
+	}
+	return body, true
+}
+
+// send unicasts msg to dst (a probe reply).
+func (n *Node) send(msg Message, dst net.Addr) {
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+		log.Printf("[beacon] Reply to %s failed: %v", dst, err)
+	}
+}
+
+// broadcast writes one message to the group once per eligible interface.
+// Writing once and letting the routing table choose would reach exactly one
+// network, which is the failure mode this protocol exists to avoid.
+func (n *Node) broadcast(msg Message) {
+	if n.pc == nil {
+		return
+	}
+	body, ok := encode(msg)
+	if !ok {
+		return
+	}
+	dst := &net.UDPAddr{IP: n.group, Port: n.cfg.Port}
+
+	n.sendMu.Lock()
+	defer n.sendMu.Unlock()
+	for i := range n.ifaces {
+		if err := n.pc.SetMulticastInterface(&n.ifaces[i]); err != nil {
+			log.Printf("[beacon] Warning: select interface %s failed: %v", n.ifaces[i].Name, err)
+			continue
+		}
+		if _, err := n.pc.WriteTo(body, nil, dst); err != nil {
+			log.Printf("[beacon] Warning: send on %s failed: %v", n.ifaces[i].Name, err)
+		}
+	}
+}
+
+// eligibleInterfaces returns every up, non-loopback, multicast-capable
+// interface that has an IPv4 address.
+func eligibleInterfaces() []net.Interface {
+	all, err := net.Interfaces()
+	if err != nil {
+		log.Printf("[beacon] Warning: could not enumerate interfaces: %v", err)
+		return nil
+	}
+
+	var out []net.Interface
+	for _, ifi := range all {
+		if ifi.Flags&net.FlagUp == 0 ||
+			ifi.Flags&net.FlagLoopback != 0 ||
+			ifi.Flags&net.FlagMulticast == 0 {
+			continue
+		}
+		addrs, err := ifi.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if ipnet, ok := a.(*net.IPNet); ok && ipnet.IP.To4() != nil {
+				out = append(out, ifi)
+				break
+			}
+		}
+	}
+	return out
+}
+
+// reusePort sets SO_REUSEADDR + SO_REUSEPORT so several listeners (a test, a
+// sidecar, a beacon v1 responder in the same netns) can bind 9099 on one host.
+func reusePort(_, _ string, c syscall.RawConn) error {
+	var serr error
+	err := c.Control(func(fd uintptr) {
+		serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
+		if serr == nil {
+			serr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return serr
+}
+
+// LocalIPv4 returns this host's first non-loopback IPv4 address, or the
+// hostname when none can be determined. Same fallback the file-based registry
+// used (discovery/service.go), kept identical so announced URLs don't shift.
+func LocalIPv4() string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, addr := range addrs {
+			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+				if ipnet.IP.To4() != nil {
+					return ipnet.IP.String()
+				}
+			}
+		}
+	}
+	host, _ := os.Hostname()
+	return host
+}

internal/meshdisco/proto.go

@@ -0,0 +1,237 @@
+// Package meshdisco implements beacon v2: local resource advertise / discover
+// over UDP multicast (239.255.99.1:9099), the protocol every meta-* service
+// and SDK-hosted plugin uses to find its neighbours.
+//
+// A node advertises Resources, each tagged with capability strings
+// ("metamesh.core", "metamesh.service/meta-sort", "metamesh.transport/nzb@1");
+// consumers keep a live view of everyone else's and filter by capability.
+// meta-core advertises "metamesh.core" carrying its /urls block, which is how
+// services locate it without a shared volume.
+//
+// The wire protocol is documented in docs/project-architecture/beacon-v2.md.
+// Keep this package and that doc in step — the Rust (meta-feeder-sdk),
+// TypeScript and Python implementations are written against the doc.
+package meshdisco
+
+import (
+	"encoding/json"
+	"strings"
+	"time"
+)
+
+const (
+	// Proto and Version form the envelope every v2 message carries. Anything
+	// else on the group (beacon v1, junk) is dropped silently.
+	Proto   = "beacon"
+	Version = 2
+
+	// DefaultGroup / DefaultPort are beacon's endpoint, on purpose: one
+	// well-known place for every kind of local resource. v1 and v2 coexist
+	// because v2 never sends v1's "discovery" / "announce" types.
+	DefaultGroup = "239.255.99.1"
+	DefaultPort  = 9099
+
+	// DefaultInterval is how often an unsolicited advertise is emitted.
+	DefaultInterval = 10 * time.Second
+
+	// LivenessFactor multiplies the interval to get the staleness window.
+	// Expiry is evaluated at read time; there is no reaper.
+	LivenessFactor = 3
+
+	// ReadBufferSize is the receive buffer; MaxDatagram is what a sender
+	// stays under so nothing fragments.
+	ReadBufferSize = 8192
+	MaxDatagram    = 1400
+
+	TypeProbe     = "probe"
+	TypeAdvertise = "advertise"
+	TypeBye       = "bye"
+
+	// CapCore is meta-core's capability; its resource carries Data.urls.
+	CapCore = "metamesh.core"
+	// CapServicePrefix + name is every service's capability.
+	CapServicePrefix = "metamesh.service/"
+	CapAnyService    = "metamesh.service/*"
+)
+
+// URLs mirrors meta-core's GET /urls response (api.URLsResponse) minus
+// redisUrl. It travels as the metamesh.core resource's Data.urls, so a service
+// locates meta-core from the advertise alone.
+type URLs struct {
+	Hostname          string `json:"hostname"`
+	BaseUrl           string `json:"baseUrl"`
+	ApiUrl            string `json:"apiUrl"`
+	WebdavUrl         string `json:"webdavUrl"`
+	WebdavUrlInternal string `json:"webdavUrlInternal"`
+}
+
+// NodeInfo says who is advertising.
+type NodeInfo struct {
+	Name     string `json:"name"`
+	Instance string `json:"instance"`
+	Version  string `json:"version,omitempty"`
+	// Status is "starting" | "running"; absent reads as running.
+	Status string `json:"status,omitempty"`
+}
+
+// Resource is one advertised thing.
+type Resource struct {
+	ID   string   `json:"id"`
+	Caps []string `json:"caps"`
+	// Endpoints: well-known names http, ui, manifest, mcp. A value starting
+	// with "/" is relative to endpoints.http.
+	Endpoints map[string]string `json:"endpoints,omitempty"`
+	Rev       string            `json:"rev,omitempty"`
+	// Binds names the one consumer instance this resource belongs to.
+	Binds string          `json:"binds,omitempty"`
+	Data  json.RawMessage `json:"data,omitempty"`
+}
+
+// Matches reports whether any cap matches pattern.
+func (r Resource) Matches(pattern string) bool {
+	for _, c := range r.Caps {
+		if CapMatches(pattern, c) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchesAny reports whether any cap matches any pattern; an empty list matches.
+func (r Resource) MatchesAny(patterns []string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+	for _, p := range patterns {
+		if r.Matches(p) {
+			return true
+		}
+	}
+	return false
+}
+
+// Endpoint returns the named endpoint as an absolute URL.
+func (r Resource) Endpoint(name string) string {
+	v := r.Endpoints[name]
+	if strings.HasPrefix(v, "/") {
+		base := r.Endpoints["http"]
+		if base == "" {
+			return ""
+		}
+		return strings.TrimRight(base, "/") + v
+	}
+	return v
+}
+
+// Message is every v2 datagram.
+type Message struct {
+	Proto     string     `json:"proto"`
+	V         int        `json:"v"`
+	Type      string     `json:"type"`
+	Node      *NodeInfo  `json:"node,omitempty"`
+	Resources []Resource `json:"resources,omitempty"`
+	// From and Want are probe-only.
+	From string   `json:"from,omitempty"`
+	Want []string `json:"want,omitempty"`
+}
+
+// Parse decodes a datagram, returning ok=false for anything that is not a
+// well-formed beacon v2 message (v1 traffic, other versions, unknown types,
+// an advertise/bye without a node).
+func Parse(b []byte) (Message, bool) {
+	var m Message
+	if err := json.Unmarshal(b, &m); err != nil {
+		return m, false
+	}
+	if m.Proto != Proto || m.V != Version {
+		return m, false
+	}
+	switch m.Type {
+	case TypeProbe:
+		return m, true
+	case TypeAdvertise, TypeBye:
+		if m.Node == nil || m.Node.Name == "" || m.Node.Instance == "" {
+			return m, false
+		}
+		return m, true
+	}
+	return m, false
+}
+
+func splitContract(s string) (string, string, bool) {
+	if i := strings.LastIndex(s, "@"); i >= 0 {
+		return s[:i], s[i+1:], true
+	}
+	return s, "", false
+}
+
+// CapMatches reports whether capability cap satisfies pattern:
+// "*" matches everything; an "@N" on the pattern must equal the cap's
+// contract (none on the pattern ignores it); "x/*" matches any variant of x;
+// otherwise the bases must be equal.
+func CapMatches(pattern, cap string) bool {
+	if pattern == "*" {
+		return true
+	}
+	pbase, pc, phas := splitContract(pattern)
+	cbase, cc, chas := splitContract(cap)
+	if phas && (!chas || pc != cc) {
+		return false
+	}
+	if prefix, ok := strings.CutSuffix(pbase, "/*"); ok {
+		rest, ok := strings.CutPrefix(cbase, prefix)
+		return ok && strings.HasPrefix(rest, "/") && len(rest) > 1
+	}
+	return pbase == cbase
+}
+
+// Neighbor is a node heard on the wire, as served by /api/neighbors.
+type Neighbor struct {
+	NodeInfo
+	Resources []Resource `json:"resources"`
+	// Caps flattens every resource's caps, for consumers that just filter.
+	Caps []string `json:"caps"`
+	// BaseUrl is the metamesh.service/* resource's endpoints.ui — kept so
+	// v1-era readers (the nav menu) keep working.
+	BaseUrl string `json:"baseUrl,omitempty"`
+	// Addr is the packet's source address, never a payload claim.
+	Addr string `json:"addr"`
+	// LastSeen is unix seconds.
+	LastSeen float64 `json:"lastSeen"`
+}
+
+// Matches reports whether any resource matches pattern.
+func (n Neighbor) Matches(pattern string) bool {
+	for _, r := range n.Resources {
+		if r.Matches(pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+func newNeighbor(node NodeInfo, resources []Resource, addr string, seen time.Time) Neighbor {
+	nb := Neighbor{
+		NodeInfo:  node,
+		Resources: resources,
+		Caps:      []string{},
+		Addr:      addr,
+		LastSeen:  float64(seen.UnixNano()) / 1e9,
+	}
+	if nb.Resources == nil {
+		nb.Resources = []Resource{}
+	}
+	seenCap := map[string]bool{}
+	for _, r := range resources {
+		for _, c := range r.Caps {
+			if !seenCap[c] {
+				seenCap[c] = true
+				nb.Caps = append(nb.Caps, c)
+			}
+		}
+		if nb.BaseUrl == "" && r.Matches(CapAnyService) {
+			nb.BaseUrl = r.Endpoint("ui")
+		}
+	}
+	return nb
+}

internal/meshdisco/proto_test.go

@@ -0,0 +1,140 @@
+package meshdisco
+
+import (
+	"encoding/json"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestCapMatches(t *testing.T) {
+	cases := []struct {
+		pattern, cap string
+		want         bool
+	}{
+		{"metamesh.transport/*", "metamesh.transport/nzb@1", true},
+		{"metamesh.transport/*", "metamesh.transport/ipfs", true},
+		{"metamesh.transport/*", "metamesh.transport", false},
+		{"metamesh.transport/*", "metamesh.transportx/a", false},
+		{"metamesh.transport/*", "metamesh.transport/", false},
+		{"metamesh.transport/nzb", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@2", true},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb@1", false},
+		{"metamesh.transport/nzb@2", "metamesh.transport/nzb", false},
+		{"metamesh.transport/*@1", "metamesh.transport/nzb@1", true},
+		{"metamesh.core", "metamesh.core", true},
+		{"metamesh.core", "metamesh.core2", false},
+		{"mcp", "mcp/x", false},
+		{"*", "anything@3", true},
+	}
+	for _, c := range cases {
+		if got := CapMatches(c.pattern, c.cap); got != c.want {
+			t.Errorf("CapMatches(%q, %q) = %v, want %v", c.pattern, c.cap, got, c.want)
+		}
+	}
+}
+
+func TestParseRejectsV1AndForeign(t *testing.T) {
+	for _, raw := range []string{
+		`{"type":"discovery"}`,
+		`{"type":"announce","name":"x","tools":[]}`,
+		`{"v":1,"type":"announce","name":"meta-sort"}`,
+		`{"proto":"beacon","v":3,"type":"probe"}`,
+		`{"proto":"beacon","v":2,"type":"announce"}`,
+		`{"proto":"beacon","v":2,"type":"advertise"}`,
+		"\x00garbage",
+	} {
+		if _, ok := Parse([]byte(raw)); ok {
+			t.Errorf("Parse(%q) accepted", raw)
+		}
+	}
+	if _, ok := Parse([]byte(`{"proto":"beacon","v":2,"type":"probe","want":["metamesh.core"]}`)); !ok {
+		t.Error("valid probe rejected")
+	}
+}
+
+func TestNeighborRowCarriesCapsAndBaseUrl(t *testing.T) {
+	nb := newNeighbor(
+		NodeInfo{Name: "meta-sort", Instance: "metasort-app"},
+		[]Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}, Endpoints: map[string]string{"ui": "https://x"}}},
+		"10.0.0.1", time.Now(),
+	)
+	if nb.BaseUrl != "https://x" || len(nb.Caps) != 1 || !nb.Matches(CapAnyService) {
+		t.Fatalf("unexpected row: %+v", nb)
+	}
+}
+
+// End to end over loopback: a probe with a non-matching want gets no reply;
+// a matching one gets the advertise; advertise then bye updates the view.
+func TestProbeAndRecord(t *testing.T) {
+	port := freePort(t)
+	n := New(Config{
+		Name: "meta-core", Instance: "core-1", Port: port, Interval: time.Hour,
+		Payload: func() (string, []Resource) {
+			return "running", []Resource{{ID: "core", Caps: []string{CapCore}}}
+		},
+	})
+	if err := n.Start(); err != nil {
+		t.Fatal(err)
+	}
+	defer n.Stop()
+
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	dst := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}
+	buf := make([]byte, ReadBufferSize)
+
+	send := func(m Message) {
+		b, _ := json.Marshal(m)
+		if _, err := c.WriteToUDP(b, dst); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{"metamesh.service/*"}})
+	c.SetReadDeadline(time.Now().Add(300 * time.Millisecond))
+	if _, _, err := c.ReadFromUDP(buf); err == nil {
+		t.Fatal("non-matching probe got a reply")
+	}
+
+	send(Message{Proto: Proto, V: Version, Type: TypeProbe, From: "cli", Want: []string{CapCore}})
+	c.SetReadDeadline(time.Now().Add(2 * time.Second))
+	nr, _, err := c.ReadFromUDP(buf)
+	if err != nil {
+		t.Fatalf("matching probe got no reply: %v", err)
+	}
+	if m, ok := Parse(buf[:nr]); !ok || m.Type != TypeAdvertise || m.Node.Instance != "core-1" {
+		t.Fatalf("bad reply: %s", buf[:nr])
+	}
+
+	peer := &NodeInfo{Name: "meta-sort", Instance: "sort-1"}
+	send(Message{Proto: Proto, V: Version, Type: TypeAdvertise, Node: peer,
+		Resources: []Resource{{ID: "service", Caps: []string{"metamesh.service/meta-sort"}}}})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 1 })
+	send(Message{Proto: Proto, V: Version, Type: TypeBye, Node: peer})
+	waitFor(t, func() bool { return len(n.Neighbors()) == 0 })
+}
+
+func freePort(t *testing.T) int {
+	c, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	return c.LocalAddr().(*net.UDPAddr).Port
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+	t.Fatal("condition not reached")
+}

internal/schema/classify.go

@@ -21,7 +21,7 @@ import (
 // (crc32: "bafk3eaqerjhiwhy").
 var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
 
-// ClassifyValue returns the (primitive, hint) for a Redis value.
+// ClassifyValue returns the (primitive, hint) for a stored value.
 // exists=false (key missing) and JSON null literal both map to PrimUndefined.
 // Empty string is a valid string.
 func ClassifyValue(value string, exists bool) (Primitive, Hint) {

internal/schema/indexer.go

@@ -6,17 +6,20 @@ import (
 	"regexp"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	"github.com/metazla/meta-core/internal/events"
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
 
-// fieldState is the per-field accumulator. perKey lets us undo a previous
-// contribution when the same Redis key is re-SET or DELed.
+// rebuildInterval is how often the schema is recomputed in the background.
+// It describes the *shape* of the data (which fields exist, their types), which
+// drifts slowly; a few minutes of staleness is invisible to the editor that
+// reads it, and POST /api/schema/rescan recomputes on demand.
+const rebuildInterval = 10 * time.Minute
+
+// fieldState is the per-field accumulator.
 type fieldState struct {
 	counts         map[string]int
 	primitives     map[Primitive]int
@@ -24,12 +27,6 @@ type fieldState struct {
 	typedCount     int
 	undefinedCount int
 	languageKeyed  bool
-	perKey         map[string]labelPair
-}
-
-type labelPair struct {
-	prim Primitive
-	hint Hint
 }
 
 func newFieldState() *fieldState {
@@ -37,19 +34,25 @@ func newFieldState() *fieldState {
 		counts:     make(map[string]int),
 		primitives: make(map[Primitive]int),
 		hintCounts: make(map[Hint]int),
-		perKey:     make(map[string]labelPair),
 	}
 }
 
-// Indexer maintains a live field-schema by consuming meta:events.
+// Indexer maintains the per-field schema by scanning the store.
+//
+// It used to follow meta:events and keep, per field, a map of EVERY record key
+// it had seen (so a re-set or delete could undo that key's contribution). At
+// 6.3M keys that map was ~2 GB of RAM — about half of meta-core's resident
+// memory once the search copy was gone — and it only worked if every event was
+// replayed into it, which is half of why meta-core republished every field on
+// boot. A periodic streaming scan holds one accumulator per FIELD instead and
+// is exact by construction.
 type Indexer struct {
-	client    *redis.Client
-	keyPrefix string
+	store *kv.Store
 
 	mu   sync.RWMutex
-	fields map[string]*fieldState
+	snap *SchemaResponse
 
-	processed uint64 // total events processed since start
+	rebuildMu sync.Mutex
 
 	ctx     context.Context
 	cancel  context.CancelFunc
@@ -58,37 +61,41 @@ type Indexer struct {
 	running bool
 }
 
-// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
-// so the indexer can build absolute Redis keys for GET calls.
-func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
+// NewIndexer creates an Indexer over store.
+func NewIndexer(store *kv.Store) *Indexer {
 	ctx, cancel := context.WithCancel(context.Background())
-	return &Indexer{
-		client:    client,
-		keyPrefix: keyPrefix,
-		fields:    make(map[string]*fieldState),
-		ctx:       ctx,
-		cancel:    cancel,
-	}
+	return &Indexer{store: store, ctx: ctx, cancel: cancel}
 }
 
-// Start begins consuming the meta:events stream from the earliest available ID.
+// Start computes the schema in the background now and every rebuildInterval.
 func (i *Indexer) Start() error {
 	i.runMu.Lock()
+	defer i.runMu.Unlock()
 	if i.running {
-		i.runMu.Unlock()
 		return nil
 	}
 	i.running = true
-	i.runMu.Unlock()
-
 	i.wg.Add(1)
-	go i.consumeLoop()
-
-	log.Println("[SchemaIndexer] Started - consuming meta:events stream")
+	go func() {
+		defer i.wg.Done()
+		ticker := time.NewTicker(rebuildInterval)
+		defer ticker.Stop()
+		for {
+			if _, err := i.Rebuild(i.ctx); err != nil && i.ctx.Err() == nil {
+				log.Printf("[SchemaIndexer] rebuild failed: %v", err)
+			}
+			select {
+			case <-i.ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+	log.Println("[SchemaIndexer] Started - scanning the store")
 	return nil
 }
 
-// Stop terminates the consumer goroutine.
+// Stop terminates the background rebuilds.
 func (i *Indexer) Stop() error {
 	i.runMu.Lock()
 	if !i.running {
@@ -104,93 +111,55 @@ func (i *Indexer) Stop() error {
 	return nil
 }
 
-// ProcessedCount returns the total number of stream events processed.
-// Used by WaitForDrain to detect when a rescan flood has been absorbed.
-func (i *Indexer) ProcessedCount() uint64 {
-	return atomic.LoadUint64(&i.processed)
-}
+// Rebuild recomputes the schema from every record key and publishes it. One
+// rebuild runs at a time; a concurrent caller waits and gets the fresh result.
+func (i *Indexer) Rebuild(ctx context.Context) (*SchemaResponse, error) {
+	i.rebuildMu.Lock()
+	defer i.rebuildMu.Unlock()
 
-// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
-// the timeout elapses. Suitable for use right after RepublishMetadata.
-func (i *Indexer) WaitForDrain(timeout time.Duration) {
-	deadline := time.Now().Add(timeout)
-	last := i.ProcessedCount()
-	stableSince := time.Now()
-	for time.Now().Before(deadline) {
-		time.Sleep(100 * time.Millisecond)
-		now := i.ProcessedCount()
-		if now != last {
-			last = now
-			stableSince = time.Now()
-			continue
-		}
-		if time.Since(stableSince) >= 300*time.Millisecond {
-			return
-		}
-	}
-}
-
-func (i *Indexer) consumeLoop() {
-	defer i.wg.Done()
-	lastID := "0"
-	for {
-		select {
-		case <-i.ctx.Done():
-			return
-		default:
-		}
-
-		res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
-			Streams: []string{events.MetaEventsStream, lastID},
-			Block:   2 * time.Second,
-			Count:   500,
-		}).Result()
-
-		if err != nil {
-			if err == redis.Nil || i.ctx.Err() != nil {
-				continue
-			}
-			log.Printf("[SchemaIndexer] XRead error: %v", err)
-			time.Sleep(1 * time.Second)
-			continue
-		}
-
-		for _, stream := range res {
-			for _, msg := range stream.Messages {
-				lastID = msg.ID
-				i.processEvent(msg.Values)
-				atomic.AddUint64(&i.processed, 1)
-			}
-		}
-	}
-}
-
-func (i *Indexer) processEvent(values map[string]any) {
-	op, _ := values["type"].(string)
-	key, _ := values["key"].(string)
-	if key == "" {
-		return
-	}
-	fieldPath, keyHint, ok := parseFieldPath(key)
+	started := time.Now()
+	fields := make(map[string]*fieldState)
+	err := i.store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
+		for _, row := range rows {
+			fieldPath, keyHint, ok := parseFieldPath(row.K)
 			if !ok {
-		return
+				continue
+			}
+			fs := fields[fieldPath]
+			if fs == nil {
+				fs = newFieldState()
+				fields[fieldPath] = fs
+			}
+			if keyHint != "" {
+				fs.languageKeyed = true
+			}
+			prim, hint := ClassifyValue(row.V, true)
+			incContribution(fs, labelPair{prim: prim, hint: hint})
+		}
+		return ctx.Err()
+	})
+	if err != nil {
+		return nil, err
 	}
 
-	switch op {
-	case "set":
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
-		cancel()
-		exists := err != redis.Nil
-		if err != nil && err != redis.Nil {
-			// Treat transient errors as "value disappeared" — nothing to record.
-			return
+	snap := &SchemaResponse{
+		Fields:      make(map[string]*FieldSchema, len(fields)),
+		GeneratedAt: time.Now().UTC(),
+		Source:      "scan",
 	}
-		prim, hint := ClassifyValue(val, exists)
-		i.upsert(fieldPath, key, keyHint, prim, hint)
-	case "del", "expire":
-		i.remove(fieldPath, key)
+	for fp, fs := range fields {
+		snap.Fields[fp] = computeFieldSchema(fs)
 	}
+	i.mu.Lock()
+	i.snap = snap
+	i.mu.Unlock()
+	log.Printf("[SchemaIndexer] %d fields from a full scan in %s", len(fields), time.Since(started).Round(time.Millisecond))
+	return snap, nil
+}
+
+type labelPair struct {
+	prim Primitive
+	hint Hint
 }
 
 // parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
@@ -222,46 +191,6 @@ func parseFieldPath(key string) (string, KeyHint, bool) {
 	return property, "", true
 }
 
-func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		fs = newFieldState()
-		i.fields[fieldPath] = fs
-	}
-	if keyHint != "" {
-		fs.languageKeyed = true
-	}
-
-	if old, ok := fs.perKey[redisKey]; ok {
-		decContribution(fs, old)
-	}
-
-	fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
-	incContribution(fs, labelPair{prim: prim, hint: hint})
-}
-
-func (i *Indexer) remove(fieldPath, redisKey string) {
-	i.mu.Lock()
-	defer i.mu.Unlock()
-
-	fs, ok := i.fields[fieldPath]
-	if !ok {
-		return
-	}
-	old, ok := fs.perKey[redisKey]
-	if !ok {
-		return
-	}
-	delete(fs.perKey, redisKey)
-	decContribution(fs, old)
-	if len(fs.perKey) == 0 {
-		delete(i.fields, fieldPath)
-	}
-}
-
 func incContribution(fs *fieldState, lp labelPair) {
 	label := breakdownLabel(lp.prim, lp.hint)
 	fs.counts[label]++
@@ -276,43 +205,15 @@ func incContribution(fs *fieldState, lp labelPair) {
 	}
 }
 
-func decContribution(fs *fieldState, lp labelPair) {
-	label := breakdownLabel(lp.prim, lp.hint)
-	fs.counts[label]--
-	if fs.counts[label] <= 0 {
-		delete(fs.counts, label)
-	}
-	if lp.prim == PrimUndefined {
-		fs.undefinedCount--
-		return
-	}
-	fs.typedCount--
-	fs.primitives[lp.prim]--
-	if fs.primitives[lp.prim] <= 0 {
-		delete(fs.primitives, lp.prim)
-	}
-	if lp.hint != "" {
-		fs.hintCounts[lp.hint]--
-		if fs.hintCounts[lp.hint] <= 0 {
-			delete(fs.hintCounts, lp.hint)
-		}
-	}
-}
-
-// Snapshot returns the current schema. Safe to call concurrently with consumption.
+// Snapshot returns the latest computed schema. Before the first scan finishes
+// it is empty with Source "pending".
 func (i *Indexer) Snapshot() *SchemaResponse {
 	i.mu.RLock()
 	defer i.mu.RUnlock()
-
-	out := &SchemaResponse{
-		Fields:      make(map[string]*FieldSchema, len(i.fields)),
-		GeneratedAt: time.Now().UTC(),
-		Source:      "live",
+	if i.snap == nil {
+		return &SchemaResponse{Fields: map[string]*FieldSchema{}, GeneratedAt: time.Now().UTC(), Source: "pending"}
 	}
-	for fp, fs := range i.fields {
-		out.Fields[fp] = computeFieldSchema(fs)
-	}
-	return out
+	return i.snap
 }
 
 func computeFieldSchema(fs *fieldState) *FieldSchema {

internal/schema/indexer_test.go

@@ -0,0 +1,56 @@
+package schema
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+func TestRebuildDerivesTypesFromTheStore(t *testing.T) {
+	store, err := kv.Open(filepath.Join(t.TempDir(), "kv.db"), kv.Options{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+	if err := store.Update(context.Background(), func(tx *kv.Tx) error {
+		for k, v := range map[string]string{
+			"file:A/sizeByte":   "100",
+			"file:B/sizeByte":   "200",
+			"file:A/title/eng":  "One",
+			"file:B/title/fre":  "Deux",
+			"file:A/mixed":      "1",
+			"file:B/mixed":      "x",
+			"cid:bafknotrecord": "A",
+		} {
+			if err := tx.Put(k, v); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	ix := NewIndexer(store)
+	if got := ix.Snapshot(); got.Source != "pending" || len(got.Fields) != 0 {
+		t.Fatalf("before a scan: %+v", got)
+	}
+	snap, err := ix.Rebuild(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if f := snap.Fields["sizeByte"]; f == nil || f.Type != PrimInt || f.Breakdown["int"] != 2 {
+		t.Fatalf("sizeByte = %+v", f)
+	}
+	if f := snap.Fields["title"]; f == nil || f.KeyHint != KeyHintLanguageCode {
+		t.Fatalf("title = %+v (want language-keyed)", f)
+	}
+	if f := snap.Fields["mixed"]; f == nil || f.Type != PrimMixed {
+		t.Fatalf("mixed = %+v", f)
+	}
+	if len(snap.Fields) != 3 {
+		t.Fatalf("fields = %v (non-record keys must not count)", snap.Fields)
+	}
+}

internal/snapshot/export.go

@@ -24,7 +24,7 @@ type ExportOptions struct {
 	SourceHost string
 }
 
-// Exporter writes snapshot ZIPs from a live Redis store.
+// Exporter writes snapshot ZIPs from the live metadata store.
 type Exporter struct {
 	storage *storage.Client
 }

internal/snapshot/flatten.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-// Flat keys use "/" as the separator. Property paths inside Redis are
+// Flat keys use "/" as the separator. Property paths inside the store are
 // stored as e.g. "fileinfo/streamdetails/video/0/codec".
 const flatSep = "/"
 
@@ -33,7 +33,7 @@ func Reconstruct(flat map[string]string) interface{} {
 }
 
 // Flatten is the inverse of Reconstruct: walks a nested value and produces a
-// flat property map whose values are all strings (Redis-ready).
+// flat property map whose values are all strings (store-ready).
 //
 // Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
 // values are stored as "" to preserve the field's presence — this matches
@@ -127,7 +127,7 @@ func flattenInto(v interface{}, path string, out map[string]string) {
 		out[path] = ""
 	case map[string]interface{}:
 		if len(t) == 0 && path != "" {
-			// Empty objects don't have a Redis representation; skip.
+			// Empty objects don't have a flat-key representation; skip.
 			return
 		}
 		for k, child := range t {

internal/snapshot/import.go

@@ -73,7 +73,7 @@ type ImportResult struct {
 	PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
 }
 
-// Importer applies snapshot ZIPs to a live Redis store.
+// Importer applies snapshot ZIPs to the live metadata store.
 type Importer struct {
 	storage *storage.Client
 }
@@ -83,7 +83,7 @@ func NewImporter(stor *storage.Client) *Importer {
 }
 
 // Import reads a ZIP from r (which must support io.ReaderAt; callers pass
-// *bytes.Reader after buffering the upload) and applies it to Redis.
+// *bytes.Reader after buffering the upload) and applies it to the store.
 func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
 	if i.storage == nil || !i.storage.IsConnected() {
 		return nil, fmt.Errorf("storage not connected")

internal/snapshot/wipe.go

@@ -16,7 +16,7 @@ type WipeResult struct {
 	MetadataDeleted int64 `json:"metadataDeleted"`
 }
 
-// Wiper clears metadata from Redis.
+// Wiper clears metadata from the store.
 type Wiper struct {
 	storage *storage.Client
 }

internal/storage/cid_resolution.go

@@ -5,10 +5,9 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/metazla/meta-core/internal/cid"
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 // This file implements the CID-resolution layer that lives on top of the
@@ -18,7 +17,7 @@ import (
 //     The UUID is an internal implementation detail; the mental model is
 //     "N CIDs → one record of keys→metadata".
 //   - Every known CID for a file is registered as a reverse-index entry:
-//       cid:<bareCid> → <uuid>                (Redis STRING)
+//       cid:<bareCid> → <uuid>
 //     and as a member of the per-root bare-CID key-set:
 //       file:<uuid>/cids/<bareCid> = "true"   (flat key, sentinel value)
 //   - A CID is self-describing: its algorithm is recoverable from the CIDv1
@@ -27,14 +26,13 @@ import (
 //     *derived by rank* from the key-set on read (GetMetadataDocument), never
 //     stored.
 //
-// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
-// any write of a cids/<cid> field and register the reverse-index entry, so
+// The record write path (writeFieldsTx) catches any write of a cids/<cid>
+// field and registers the reverse-index entry in the same transaction, so
 // writers that just emit the key-set members don't need to know about the
-// reverse index at all. The hook helpers are *Locked — they assume c.mu is
-// already held by the caller, avoiding nested RLock (which can deadlock
-// against a waiting writer).
+// reverse index at all. The helpers here take the caller's *kv.Tx: a write
+// must never open a second transaction (there is one writer connection).
 
-// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
+// CIDIndexPrefix is the key prefix for the reverse index. Combined
 // with a bare CID, it forms the full key: "cid:bafk…".
 const CIDIndexPrefix = "cid:"
 
@@ -45,116 +43,112 @@ const CIDIndexPrefix = "cid:"
 // reverse-index entries that need cleanup).
 const CIDsKeyPrefix = "cids/"
 
-// DuplicatesField is the per-root SET key holding additional file paths
-// whose content hashes to the same CIDs as this root. The watcher writes
-// here when it sees a new path for content that's already indexed; meta-dup
-// reads from it.
-const DuplicatesField = "duplicates"
-
-// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
-// cidStr is a bare multibase CIDv1 ("bafk…").
-func (c *Client) buildCIDIndexKey(cidStr string) string {
-	return c.buildKey(CIDIndexPrefix + cidStr)
-}
+// Duplicate paths — other physical locations whose content hashes to the same
+// CIDs as a root — live as dup:<rid>/<path> = "" (a key-set). The watcher
+// writes one when it sees a new path for content that's already indexed.
 
 // Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
-// in the file index. Returns the new UUID. The caller is expected to follow
+// in the root index. Returns the new UUID. The caller is expected to follow
 // up with AddAlias for at least one CID — a root without any alias is
 // reachable only via the index, which works but defeats the point.
 func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return "", fmt.Errorf("not connected")
-	}
+	var uuid string
+	err := c.update(func(tx *kv.Tx) error {
+		var err error
+		uuid, err = mintTx(tx, filePath, size, mtimeNano)
+		return err
+	})
+	return uuid, err
+}
 
+func mintTx(tx *kv.Tx, filePath string, size, mtimeNano int64) (string, error) {
 	uuid, err := NewUUID()
 	if err != nil {
 		return "", fmt.Errorf("uuid: %w", err)
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	prefix := c.buildKeyPrefix(uuid)
-	pipe := c.client.Pipeline()
-	pipe.Set(ctx, prefix+"filePath", filePath, 0)
-	pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
-	pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
-	pipe.SAdd(ctx, c.buildIndexKey(), uuid)
-	c.addFieldsToPipe(ctx, pipe, uuid, []string{"filePath", "sizeByte", "mtimeNano"})
-	if _, err := pipe.Exec(ctx); err != nil {
-		return "", fmt.Errorf("mint pipeline: %w", err)
-	}
-	return uuid, nil
+	return uuid, writeFieldsTx(tx, uuid, map[string]string{
+		"filePath":  filePath,
+		"sizeByte":  strconv.FormatInt(size, 10),
+		"mtimeNano": strconv.FormatInt(mtimeNano, 10),
+	})
 }
 
 // AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
 // index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
 // Idempotent: registering the same CID twice is a no-op.
 func (c *Client) AddAlias(uuid, cidStr string) error {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	return c.addAliasLocked(ctx, uuid, cidStr)
-}
-
-// addAliasLocked is the lock-free body of AddAlias. The caller must already
-// hold c.mu (read or write). Split out so the alias-maintenance hooks
-// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
-// the work without re-entering the lock — nested RLock can deadlock against
-// a waiting writer.
-func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
 	if uuid == "" || cidStr == "" {
 		return fmt.Errorf("uuid and cid are required")
 	}
+	return c.update(func(tx *kv.Tx) error {
+		return writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + cidStr: "true"})
+	})
+}
 
-	keysetField := CIDsKeyPrefix + cidStr
-	keysetKey := c.buildKeyPrefix(uuid) + keysetField
-
-	// Every branch below writes the key-set member, so index the field name
-	// once here — including the two guard branches that skip the alias itself.
-	// Missing it would leave `cids/<cid>` invisible to a record read.
-	defer c.addFieldLocked(ctx, uuid, keysetField)
-
+// addAliasTx registers the reverse-index entry for a cids/<cid> member the
+// caller is writing (writeFieldsTx writes the member itself).
+func addAliasTx(tx *kv.Tx, uuid, cidStr string) error {
+	if uuid == "" || cidStr == "" {
+		return fmt.Errorf("uuid and cid are required")
+	}
 	// Self-pointing alias guard. If the caller is trying to register
 	// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
 	// dual-root failure mode in disguise — it means somebody wrote
 	// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
-	// write; leave any existing alias alone. Still record the key-set
-	// membership so the field-level write succeeds.
+	// write; leave any existing alias alone. The key-set membership is still
+	// recorded by the caller, so the field-level write succeeds.
 	if uuid == cidStr {
-		return c.client.Set(ctx, keysetKey, "true", 0).Err()
+		return nil
 	}
-
 	// Guard against the dual-root pattern: if cid:<cid> already points at a
-	// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
-	// just record the key-set membership so it's still discoverable via the
-	// root → cids enumeration.
-	indexKey := c.buildCIDIndexKey(cidStr)
-	existing, err := c.client.Get(ctx, indexKey).Result()
-	if err != nil && err != redis.Nil {
+	// DIFFERENT uuid, don't silently overwrite. Leave the existing alias; the
+	// key-set membership keeps the CID discoverable via root → cids.
+	indexKey := CIDIndexPrefix + cidStr
+	existing, _, err := tx.Get(tx.Context(), indexKey)
+	if err != nil {
 		return fmt.Errorf("read existing alias: %w", err)
 	}
 	if existing != "" && existing != uuid {
-		return c.client.Set(ctx, keysetKey, "true", 0).Err()
-	}
-
-	pipe := c.client.Pipeline()
-	pipe.Set(ctx, indexKey, uuid, 0)
-	pipe.Set(ctx, keysetKey, "true", 0)
-	if _, err := pipe.Exec(ctx); err != nil {
-		return fmt.Errorf("add alias pipeline: %w", err)
-	}
 		return nil
+	}
+	if existing == uuid {
+		return nil
+	}
+	return tx.Put(indexKey, uuid)
+}
+
+// removeAliasTx is the inverse of addAliasTx: it drops the reverse-index
+// entry cid:<cid> → <uuid>. The caller deletes the cids/<cid> key-set member
+// itself (DeleteProperty does both).
+//
+// ⚠ Why this exists. A cids/ member write touches two keys — the flat
+// file:<uuid>/cids/<cid> and this reverse alias — while the delete path once
+// only removed the first. A "deleted" CID therefore still resolved: GET
+// /meta/<cid>, ResolveRoot and GetByCID all kept answering with the record.
+// The asymmetry is not theoretical. meta-share published dag-pb roots computed
+// over RAR volumes and par2 files onto six records on watch.nsl.sh; removing
+// the member was expressible, removing the claim was not, so the false
+// identity survived its own retraction.
+//
+// Guarded, not blind: the alias is removed only when it currently points at
+// THIS uuid. Two records can legitimately race to claim a cid (see the
+// dual-root guard in addAliasTx, which leaves the first writer's alias in
+// place); deleting a member from the loser must not unpick the winner's index.
+func removeAliasTx(tx *kv.Tx, uuid, cidStr string) error {
+	if uuid == "" || cidStr == "" {
+		return nil
+	}
+	indexKey := CIDIndexPrefix + cidStr
+	existing, ok, err := tx.Get(tx.Context(), indexKey)
+	if err != nil {
+		return fmt.Errorf("read existing alias: %w", err)
+	}
+	if !ok || existing != uuid {
+		// Absent, or somebody else owns this alias. Leave it.
+		return nil
+	}
+	_, err = tx.Delete(indexKey)
+	return err
 }
 
 // ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
@@ -174,8 +168,8 @@ func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error
 //   - hash matches a reverse-index entry → return UUID
 //   - otherwise return hash unchanged (legacy / explicit-UUID writes)
 //
-// Errors are converted to "no resolution" — if Redis is down or the lookup
-// fails we let the caller proceed with the bare hash and surface the Redis
+// Errors are converted to "no resolution" — if the store is down or the
+// lookup fails we let the caller proceed with the bare hash and surface the
 // failure on the actual operation.
 func (c *Client) ResolveRoot(hash string) string {
 	if hash == "" {
@@ -192,23 +186,16 @@ func (c *Client) ResolveRoot(hash string) string {
 // when the CID is unknown; the empty string is the "not found" signal rather
 // than a typed error so callers can branch without errors.As.
 func (c *Client) GetByCID(cidStr string) (string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return "", fmt.Errorf("not connected")
+	s, err := c.kv()
+	if err != nil {
+		return "", err
 	}
 	if cidStr == "" {
 		return "", nil
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := opCtx()
 	defer cancel()
-
-	uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
-	if err == redis.Nil {
-		return "", nil
-	}
+	uuid, _, err := s.Get(ctx, CIDIndexPrefix+cidStr)
 	if err != nil {
 		return "", fmt.Errorf("get cid index: %w", err)
 	}
@@ -216,131 +203,49 @@ func (c *Client) GetByCID(cidStr string) (string, error) {
 }
 
 // AddDuplicatePath records that path is another physical location whose
-// content matches uuid. Returns true if path was newly added (not already
-// in the set). The watcher calls this when it sees a file whose midhash
-// matches an existing root but at a different filePath.
+// content matches uuid. Returns true if path was newly added. The watcher
+// calls this when it sees a file whose midhash matches an existing root but
+// at a different filePath.
 func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return false, fmt.Errorf("not connected")
-	}
 	if uuid == "" || path == "" {
 		return false, nil
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
-	if err != nil {
-		return false, fmt.Errorf("sadd duplicates: %w", err)
-	}
-	return added > 0, nil
+	added := false
+	err := c.update(func(tx *kv.Tx) error {
+		var err error
+		added, err = tx.PutIfAbsent(dupPrefix(uuid)+path, "")
+		return err
+	})
+	return added, err
 }
 
-// cidsForRootLocked returns a root's bare-CID key-set members (with the cids/
-// prefix stripped). Caller must hold c.mu.
-//
-// Reads the field index and filters by prefix; only an un-indexed root falls
-// back to a SCAN, and then over the narrower cids/ glob. See field_index.go.
-func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
-	return c.fieldsWithPrefixLocked(ctx, uuid, CIDsKeyPrefix)
+// cidsForRoot returns a root's bare-CID key-set members (cids/ prefix
+// stripped).
+func cidsForRoot(ctx context.Context, r kv.Reader, uuid string) ([]string, error) {
+	prefix := recordPrefix(uuid) + CIDsKeyPrefix
+	rows, err := r.Range(ctx, prefix, "", 0)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]string, 0, len(rows))
+	for _, row := range rows {
+		out = append(out, row.K[len(prefix):])
+	}
+	return out, nil
 }
 
 // DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
 // (including the cids/<cid> key-set), every reverse-index entry for those
-// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
-// is a successful no-op.
+// CIDs, its duplicates, search row and index membership. Idempotent — a uuid
+// with no remaining keys is a successful no-op.
 func (c *Client) DeleteRoot(uuid string) error {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return fmt.Errorf("not connected")
-	}
 	if uuid == "" {
 		return nil
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	prefix := c.buildKeyPrefix(uuid)
-
-	// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
-	//    the list of reverse-index entries to clean up.
-	cids, err := c.cidsForRootLocked(ctx, uuid)
-	if err != nil {
+	return c.update(func(tx *kv.Tx) error {
+		_, err := deleteRecordTx(tx, uuid)
 		return err
-	}
-
-	// 2. Delete every reverse-index entry in one shot.
-	if len(cids) > 0 {
-		keys := make([]string, 0, len(cids))
-		for _, cidStr := range cids {
-			keys = append(keys, c.buildCIDIndexKey(cidStr))
-		}
-		if err := c.client.Del(ctx, keys...).Err(); err != nil {
-			return fmt.Errorf("del cid index: %w", err)
-		}
-	}
-
-	// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return fmt.Errorf("scan root keys: %w", err)
-		}
-		if len(keys) > 0 {
-			if err := c.client.Del(ctx, keys...).Err(); err != nil {
-				return fmt.Errorf("del root keys: %w", err)
-			}
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-
-	// 4. Remove from the file index set.
-	if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
-		return fmt.Errorf("srem index: %w", err)
-	}
-	// Marked so the incremental search-index refresh reads it, finds it gone,
-	// and evicts it — otherwise a deleted root keeps answering searches until
-	// the next full rebuild.
-	c.markDirtyLocked(ctx, uuid)
-	return nil
-}
-
-// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
-// root. If it's a key-set member — field name starts with "cids/" — it
-// registers the implied CID in the reverse index. Writers emit cids/<cid>
-// members without knowing about the reverse index; this hook keeps it in
-// sync transparently.
-//
-// Caller must hold c.mu. Errors are intentionally swallowed at the call site
-// (logged, not returned): an out-of-sync reverse index is recoverable via a
-// sweep, while a failed property write is not.
-func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
-	cidStr := cidFromKeysetField(field)
-	if cidStr == "" {
-		return nil
-	}
-	return c.addAliasLocked(ctx, uuid, cidStr)
-}
-
-// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
-// over every entry in a metadata map. Used by SetMetadataFlat /
-// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
-// hold c.mu.
-func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
-	for k := range metadata {
-		_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
-	}
+	})
 }
 
 // cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
@@ -370,24 +275,23 @@ type MetadataDocument struct {
 // the duplicates SET. Returns nil if the root has no keys at all (deleted or
 // never existed).
 func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
+	s, err := c.kv()
+	if err != nil {
+		return nil, err
 	}
 	if uuid == "" {
 		return nil, nil
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := opCtx()
 	defer cancel()
 
-	flat, err := c.getMetadataFlatInternal(ctx, uuid)
+	flat, err := readRecord(ctx, s, uuid)
 	if err != nil {
 		return nil, err
 	}
-	prefix := c.buildKeyPrefix(uuid)
+	if flat == nil {
+		flat = make(map[string]string)
+	}
 
 	// Hoist the cids/<cid> key-set members into a typed array and derive the
 	// canonical CID by rank. The members remain in Flat too — that's how
@@ -407,9 +311,13 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
 		}
 	}
 
-	dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
-	if err != nil && err != redis.Nil {
-		return nil, fmt.Errorf("smembers duplicates: %w", err)
+	dupRows, err := s.Range(ctx, dupPrefix(uuid), "", 0)
+	if err != nil {
+		return nil, fmt.Errorf("read duplicates: %w", err)
+	}
+	dups := make([]string, 0, len(dupRows))
+	for _, row := range dupRows {
+		dups = append(dups, row.K[len(dupPrefix(uuid)):])
 	}
 
 	if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
@@ -438,34 +346,50 @@ func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhas
 	if midhash == "" {
 		return "", false, fmt.Errorf("midhash is required")
 	}
-	uuid, err := c.GetByCID(midhash)
+	var (
+		uuid   string
+		minted bool
+	)
+	// One transaction: the lookup and the mint cannot interleave with another
+	// registration of the same content, so two concurrent scans of one file
+	// can no longer mint two roots for it.
+	err := c.update(func(tx *kv.Tx) error {
+		ctx := tx.Context()
+		existing, _, err := tx.Get(ctx, CIDIndexPrefix+midhash)
 		if err != nil {
-		return "", false, fmt.Errorf("GetByCID(%s): %w", midhash, err)
+			return fmt.Errorf("GetByCID(%s): %w", midhash, err)
 		}
-
+		uuid = existing
 		if uuid == "" {
-		uuid, err = c.Mint(absPath, size, mtimeNano)
+			if uuid, err = mintTx(tx, absPath, size, mtimeNano); err != nil {
+				return fmt.Errorf("Mint(%s): %w", absPath, err)
+			}
+			minted = true
+			if err := writeFieldsTx(tx, uuid, map[string]string{CIDsKeyPrefix + midhash: "true"}); err != nil {
+				return fmt.Errorf("cids/%s for %s: %w", midhash, uuid, err)
+			}
+			return nil
+		}
+		current, _, err := tx.Get(ctx, recordPrefix(uuid)+"filePath")
 		if err != nil {
-			return "", false, fmt.Errorf("Mint(%s): %w", absPath, err)
+			return err
 		}
-		if err := c.SetProperty(uuid, CIDsKeyPrefix+midhash, "true"); err != nil {
-			return uuid, true, fmt.Errorf("SetProperty cids/%s for %s: %w", midhash, uuid, err)
+		if current != absPath {
+			if _, err := tx.PutIfAbsent(dupPrefix(uuid)+absPath, ""); err != nil {
+				return fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
 			}
-		return uuid, true, nil
+			return nil
 		}
-
-	existing, _ := c.GetProperty(uuid, "filePath")
-	if existing != absPath {
-		if _, err := c.AddDuplicatePath(uuid, absPath); err != nil {
-			return uuid, false, fmt.Errorf("AddDuplicatePath %s @ %s: %w", uuid, absPath, err)
-		}
-		return uuid, false, nil
-	}
-	if _, err := c.MergeMetadataFlat(uuid, map[string]string{
+		if err := writeFieldsTx(tx, uuid, map[string]string{
 			"sizeByte":  strconv.FormatInt(size, 10),
 			"mtimeNano": strconv.FormatInt(mtimeNano, 10),
 		}); err != nil {
-		return uuid, false, fmt.Errorf("refresh tuple for %s: %w", uuid, err)
+			return fmt.Errorf("refresh tuple for %s: %w", uuid, err)
 		}
-	return uuid, false, nil
+		return nil
+	})
+	if err != nil {
+		return "", false, err
+	}
+	return uuid, minted, nil
 }

internal/storage/cid_retraction_test.go

@@ -0,0 +1,159 @@
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+// Tests for retracting a `cids/<cid>` key-set member (cid_resolution.go's
+// removeAliasTx, reached through DeleteProperty).
+//
+// Why this is worth its own file. Adding a member writes two keys — the flat
+// `file:<uuid>/cids/<cid>` and the reverse alias `cid:<cid>` → `<uuid>`. Until
+// removeAliasLocked (now removeAliasTx) existed, deleting removed only the
+// first, so a retracted CID kept resolving to the record: the member was gone
+// from the document and the claim was not.
+//
+// That asymmetry had a live consequence. meta-share minted dag-pb roots and
+// midhash256s over whatever `archive::pick_media` elected, which on an
+// obfuscated posting was an arbitrary RAR volume or a par2 file. Six records on
+// watch.nsl.sh carry such members. `METADATA_KEYS.md` §2 defines a `cids/`
+// member as a digest of *this record's file* and notes that every member is a
+// reverse index back to the record — so those entries assert that an archive
+// volume is the episode. Retracting one has to remove the assertion, not just
+// the row.
+
+// aliasTarget returns the uuid that cid:<cidStr> currently resolves to, or "".
+func aliasTarget(t *testing.T, c *Client, cidStr string) string {
+	t.Helper()
+	got, _, err := c.Store().Get(context.Background(), CIDIndexPrefix+cidStr)
+	if err != nil {
+		t.Fatalf("get alias: %v", err)
+	}
+	return got
+}
+
+func hasField(t *testing.T, c *Client, hashID, field string) bool {
+	t.Helper()
+	fields, err := c.GetMetadataFlat(hashID)
+	if err != nil {
+		t.Fatalf("read record: %v", err)
+	}
+	_, ok := fields[field]
+	return ok
+}
+
+// The real shape: `…glnrqztt…` is the nzb-release locator whose record published
+// a dag-pb root computed over volume 3 of a 35-volume RAR5 set.
+const (
+	poisonedRoot = "bafybeicicuzdvuztca2hp2g2k7scgr4nnibilzgoppljeprphybqot7v6i"
+	goodMidhash  = "bagacbabaed6uiulnv7tscpxne22eztkjvnxg5almn43dpfecppixarfe7xhlu"
+)
+
+func TestRetractingACidMemberRemovesTheClaimNotJustTheRow(t *testing.T) {
+	withClient(t, func(t *testing.T, c *Client) {
+		const uuid = "01KSBHMTESTRECORD"
+
+		if err := c.SetMetadataFlat(uuid, map[string]string{
+			"cids/" + poisonedRoot: "true",
+			"cids/" + goodMidhash:  "true",
+			"fileType":             "video",
+		}); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+
+		// All three writes landed, and the CID resolves to the record.
+		if got := aliasTarget(t, c, poisonedRoot); got != uuid {
+			t.Fatalf("alias should point at the record before retraction, got %q", got)
+		}
+		if !hasField(t, c, uuid, "cids/"+poisonedRoot) {
+			t.Fatal("field index should list the member before retraction")
+		}
+		if got := c.ResolveRoot(poisonedRoot); got != uuid {
+			t.Fatalf("ResolveRoot should find the record before retraction, got %q", got)
+		}
+
+		if err := c.DeleteProperty(uuid, "cids/"+poisonedRoot); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+
+		// 1. the flat key
+		if v, ok, _ := c.Store().Get(context.Background(),
+			recordPrefix(uuid)+"cids/"+poisonedRoot); ok {
+			t.Fatalf("flat key should be gone, still %q", v)
+		}
+		// 2. the record read
+		if hasField(t, c, uuid, "cids/"+poisonedRoot) {
+			t.Fatal("record read still lists the retracted member")
+		}
+		// 3. the reverse alias — the one that used to survive
+		if got := aliasTarget(t, c, poisonedRoot); got != "" {
+			t.Fatalf("reverse alias survived retraction, still points at %q", got)
+		}
+		if got := c.ResolveRoot(poisonedRoot); got == uuid {
+			t.Fatal("a retracted cid must not resolve to the record any more")
+		}
+
+		// The siblings are untouched: retraction is surgical, not a purge.
+		if got := aliasTarget(t, c, goodMidhash); got != uuid {
+			t.Fatalf("an unrelated sibling lost its alias, got %q", got)
+		}
+		if !hasField(t, c, uuid, "cids/"+goodMidhash) {
+			t.Fatal("an unrelated sibling lost its field-index entry")
+		}
+		if v, err := c.GetProperty(uuid, "fileType"); err != nil || v != "video" {
+			t.Fatalf("an unrelated field changed: %q %v", v, err)
+		}
+	})
+}
+
+// Two records can race to claim one cid; addAliasLocked leaves the first
+// writer's alias in place. Retracting the loser's member must not unpick the
+// winner's index — which is why removeAliasLocked compares before deleting.
+func TestRetractingDoesNotStealAnotherRecordsAlias(t *testing.T) {
+	withClient(t, func(t *testing.T, c *Client) {
+		const winner = "01KSBHMWINNER"
+		const loser = "01KSBHMLOSER"
+		const shared = "bafybeiaeb2lqi4z3np3q4tarldtyjp2t5lvnoodmtpvpi2tz73azn5td2m"
+
+		if err := c.SetMetadataFlat(winner, map[string]string{"cids/" + shared: "true"}); err != nil {
+			t.Fatalf("seed winner: %v", err)
+		}
+		if err := c.SetMetadataFlat(loser, map[string]string{"cids/" + shared: "true"}); err != nil {
+			t.Fatalf("seed loser: %v", err)
+		}
+		if got := aliasTarget(t, c, shared); got != winner {
+			t.Fatalf("the first writer should own the alias, got %q", got)
+		}
+
+		if err := c.DeleteProperty(loser, "cids/"+shared); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+
+		if got := aliasTarget(t, c, shared); got != winner {
+			t.Fatalf("retracting the loser's member stole the winner's alias, now %q", got)
+		}
+		if hasField(t, c, loser, "cids/"+shared) {
+			t.Fatal("the loser's own member should still be gone")
+		}
+	})
+}
+
+// A non-cids field has no alias to clean up, and the delete must not invent one.
+func TestRetractingAnOrdinaryFieldTouchesNoAlias(t *testing.T) {
+	withClient(t, func(t *testing.T, c *Client) {
+		const uuid = "01KSBHMORDINARY"
+		if err := c.SetMetadataFlat(uuid, map[string]string{
+			"cids/" + goodMidhash: "true",
+			"quality":             "1080p",
+		}); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+		if err := c.DeleteProperty(uuid, "quality"); err != nil {
+			t.Fatalf("DeleteProperty: %v", err)
+		}
+		if got := aliasTarget(t, c, goodMidhash); got != uuid {
+			t.Fatalf("deleting an ordinary field disturbed a cid alias, got %q", got)
+		}
+	})
+}

internal/storage/clear_test.go

@@ -1,67 +1,74 @@
 package storage
 
 import (
+	"context"
+	"path/filepath"
+	"strings"
 	"testing"
 
-	"github.com/alicebob/miniredis/v2"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
-// newTestClient spins an in-memory Redis and returns a connected Client with
-// the given namespace prefix. The server is torn down with the test.
-func newTestClient(t *testing.T, prefix string) (*Client, *miniredis.Miniredis) {
+// newTestClient opens a fresh database in the test's temp dir.
+func newTestClient(t *testing.T) *Client {
 	t.Helper()
-
-	s := miniredis.RunT(t)
-	c := NewClient(prefix)
-	if err := c.Connect("redis://" + s.Addr()); err != nil {
-		t.Fatalf("connect to miniredis: %v", err)
+	c := NewClient()
+	if err := c.Connect(filepath.Join(t.TempDir(), "meta-core.db")); err != nil {
+		t.Fatalf("connect: %v", err)
 	}
 	t.Cleanup(func() { _ = c.Close() })
-	return c, s
+	return c
+}
+
+// allKeys lists every row (test helper; full scan).
+func allKeys(t *testing.T, c *Client) []string {
+	t.Helper()
+	var keys []string
+	if err := c.Store().ScanPrefix(context.Background(), "", 1000, func(rows []kv.KV) error {
+		for _, r := range rows {
+			keys = append(keys, r.K)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("scan: %v", err)
+	}
+	return keys
 }
 
 // TestClearAllMetadata_PurgesCIDReverseIndex is the regression test for the
-// leak: ClearAllMetadata used to delete every file:<root>/* key and the
-// file:__index__ set, but left the cid:<cid> → root reverse-index entries
-// behind. Those stale aliases are not cosmetic — ResolveRoot consults them, so
-// a surviving entry silently redirects a later write into a deleted root.
+// leak: ClearAllMetadata used to delete every record key and the root index,
+// but left the cid:<cid> → root reverse-index entries behind. Those stale
+// aliases are not cosmetic — ResolveRoot consults them, so a surviving entry
+// silently redirects a later write into a deleted root.
 func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
-	for _, prefix := range []string{"", "mm:"} {
-		name := "no-prefix"
-		if prefix != "" {
-			name = "prefixed"
-		}
-		t.Run(name, func(t *testing.T) {
-			c, s := newTestClient(t, prefix)
+	c := newTestClient(t)
 
-			// Two records, each with several CID aliases (a midhash plus
-			// siblings) — exactly what the watcher produces.
+	// Two records, each with several CID aliases (a midhash plus siblings) —
+	// exactly what the watcher produces.
 	aliases := map[string][]string{}
 	for i, f := range []string{"watch/a.mkv", "watch/b.mkv"} {
 		uuid, err := c.Mint(f, int64(100+i), int64(1000+i))
 		if err != nil {
 			t.Fatalf("mint: %v", err)
 		}
-				cids := []string{
-					"bafkmid" + uuid,
-					"bafkfull" + uuid,
-				}
+		cids := []string{"bafkmid" + uuid, "bafkfull" + uuid}
 		for _, cidStr := range cids {
 			if err := c.AddAlias(uuid, cidStr); err != nil {
 				t.Fatalf("add alias %s: %v", cidStr, err)
 			}
 		}
+		if _, err := c.AddDuplicatePath(uuid, "watch/copy-"+f); err != nil {
+			t.Fatalf("add duplicate: %v", err)
+		}
 		aliases[uuid] = cids
 	}
 
-			// An entry already orphaned by a previous (buggy) clear: a
-			// reverse-index key whose root no longer exists. The sweep is
-			// expected to collect it too.
-			if err := s.Set(c.buildCIDIndexKey("bafkorphan"), "01OLDDEADROOT"); err != nil {
+	// An entry already orphaned by a previous (buggy) clear: a reverse-index
+	// key whose root no longer exists. The sweep is expected to collect it too.
+	if err := c.SetRawValue(CIDIndexPrefix+"bafkorphan", "01OLDDEADROOT"); err != nil {
 		t.Fatalf("seed orphan: %v", err)
 	}
 
-			// Sanity: the aliases resolve before the clear.
 	for uuid, cids := range aliases {
 		for _, cidStr := range cids {
 			if got := c.ResolveRoot(cidStr); got != uuid {
@@ -74,19 +81,25 @@ func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
 	if err != nil {
 		t.Fatalf("ClearAllMetadata: %v", err)
 	}
-			// deletedCount counts files, not keys.
 	if deleted != 2 {
 		t.Errorf("deletedCount = %d, want 2 (files, not keys)", deleted)
 	}
 
-			// No reverse-index key may survive — not for the cleared records,
-			// not the pre-existing orphan.
-			if leaked := s.Keys(); len(leaked) != 0 {
-				t.Errorf("keys survived the clear: %v", leaked)
+	// Nothing record-shaped may survive: not the records, not their aliases,
+	// duplicates or search rows, not the pre-existing orphan. (The event
+	// streams and the schema sentinel are not metadata and stay.)
+	for _, k := range allKeys(t, c) {
+		for _, ns := range []string{recordNS, CIDIndexPrefix, rootsNS, ftsNS, dupNS} {
+			if strings.HasPrefix(k, ns) {
+				t.Errorf("key survived the clear: %q", k)
+			}
+		}
+	}
+	var ftsRows int
+	if err := c.Store().DB().QueryRow(`SELECT count(*) FROM fts`).Scan(&ftsRows); err != nil || ftsRows != 0 {
+		t.Errorf("fts rows after clear = %d (err %v), want 0", ftsRows, err)
 	}
 
-			// And the aliases must no longer resolve: ResolveRoot returns the
-			// hash unchanged when there is no index entry.
 	for _, cids := range aliases {
 		for _, cidStr := range cids {
 			if got := c.ResolveRoot(cidStr); got != cidStr {
@@ -97,15 +110,13 @@ func TestClearAllMetadata_PurgesCIDReverseIndex(t *testing.T) {
 	if got, err := c.GetByCID("bafkorphan"); err != nil || got != "" {
 		t.Errorf("post-clear GetByCID(bafkorphan) = %q (err %v), want \"\"", got, err)
 	}
-		})
-	}
 }
 
 // TestClearAllMetadata_LeavesForeignKeysAlone guards the sweep's blast radius:
-// it must key off buildCIDIndexKey (namespace-aware), not a hardcoded "cid:",
-// so a clear on a prefixed client cannot reach another namespace's index.
+// keys outside the metadata namespaces (magnet cache, coverage markers, a key
+// that merely CONTAINS "cid:") must survive a metadata wipe.
 func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
-	c, s := newTestClient(t, "mm:")
+	c := newTestClient(t)
 
 	uuid, err := c.Mint("watch/a.mkv", 100, 1000)
 	if err != nil {
@@ -114,25 +125,28 @@ func TestClearAllMetadata_LeavesForeignKeysAlone(t *testing.T) {
 	if err := c.AddAlias(uuid, "bafkmine"); err != nil {
 		t.Fatalf("add alias: %v", err)
 	}
-
-	// Another namespace's reverse-index entry, plus an unrelated key.
-	if err := s.Set("other:cid:bafktheirs", "01THEIRROOT"); err != nil {
-		t.Fatalf("seed foreign cid: %v", err)
+	for k, v := range map[string]string{"other:cid:bafktheirs": "01THEIRROOT", "btmagnet:abc": "magnet:?xt=urn:btih:abc", "unrelated": "keep-me"} {
+		if err := c.SetRawValue(k, v); err != nil {
+			t.Fatalf("seed %s: %v", k, err)
 		}
-	if err := s.Set("unrelated", "keep-me"); err != nil {
-		t.Fatalf("seed unrelated: %v", err)
 	}
 
 	if _, err := c.ClearAllMetadata(); err != nil {
 		t.Fatalf("ClearAllMetadata: %v", err)
 	}
 
-	for _, k := range []string{"other:cid:bafktheirs", "unrelated"} {
-		if !s.Exists(k) {
-			t.Errorf("clear deleted foreign key %q — the cid sweep is not namespace-aware", k)
+	for _, k := range []string{"other:cid:bafktheirs", "btmagnet:abc", "unrelated"} {
+		if _, _, ok, _ := c.GetKeyInfo(k); !ok {
+			t.Errorf("clear deleted foreign key %q", k)
 		}
 	}
-	if s.Exists(c.buildCIDIndexKey("bafkmine")) {
+	if _, _, ok, _ := c.GetKeyInfo(CIDIndexPrefix + "bafkmine"); ok {
 		t.Errorf("own reverse-index entry survived the clear")
 	}
 }
+
+// withClient runs fn against a fresh client.
+func withClient(t *testing.T, fn func(t *testing.T, c *Client)) {
+	t.Helper()
+	fn(t, newTestClient(t))
+}

internal/storage/domain_screen_migration.go

@@ -1,193 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"time"
-)
-
-// The one-shot corpus sweep for METADATA_KEYS.md §14.17: the `film` + `tv`
-// domains merged into `screen`, and the standalone-vs-serial split they were
-// really carrying moved to the new `workForm` key.
-//
-// # Why a sweep and not a read-side alias
-//
-// `domain` is **stored, not derived on read** — that is what lets the
-// `contentKind` vocabulary stay open (a peer that has never heard of a kind
-// still routes the record correctly). The price of storing it is that a
-// vocabulary change has to be paid once, in the data: every consumer's filter
-// is an exact match, so a record left saying `film` is invisible to a
-// `domain:screen` wall, and teaching every consumer to accept both spellings
-// forever is exactly the alternation the merge exists to delete.
-//
-// So the values are rewritten at rest, here, and no alias vocabulary survives
-// in query filters. (meta-search additionally normalises on the way into its
-// index, because a record can still arrive from a peer that has not been swept
-// — that is a courtesy for un-swept peers, not a second source of truth.)
-//
-// # Contract
-//
-// Idempotent: a record already carrying `screen` and a `workForm` is skipped,
-// so the endpoint is safe to call repeatedly and safe to call on a fresh box.
-// Returns the number of records **changed**, not the number examined.
-
-// legacyDomainReplacements maps a retired `domain` value to its replacement.
-var legacyDomainReplacements = map[string]string{
-	"film": "screen",
-	"tv":   "screen",
-}
-
-// workFormByContentKind is the `contentKind` → `workForm` table from
-// METADATA_KEYS.md §1.
-//
-// ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain` (and of
-// meta-search's copy). It exists here only to backfill the pre-merge corpus —
-// meta-core never classifies a record it is given, and nothing else in Go reads
-// it. If the tables ever disagree, the writers win: this one is a one-shot.
-var workFormByContentKind = map[string]string{
-	// screen
-	"movie":   "standalone",
-	"series":  "serial",
-	"episode": "serial",
-	// music — an album is a closed work, not an ongoing run.
-	"track":      "standalone",
-	"album":      "standalone",
-	"artist":     "standalone",
-	"musicVideo": "standalone",
-	"djMix":      "standalone",
-	"liveSet":    "standalone",
-	// literature
-	"book":      "standalone",
-	"audiobook": "standalone",
-	"comic":     "serial",
-	"manga":     "serial",
-	"magazine":  "serial",
-	// science
-	"paper": "standalone",
-	// ⚠ `pack` is deliberately absent: a season pack is `serial`, an album
-	// release `standalone`, and only the writer knows which. A pack that
-	// reaches this sweep without a `workForm` keeps none — inventing one would
-	// be a guess written permanently into the record.
-}
-
-// domainScreenPlan is one record's pending rewrite. Empty strings mean "leave
-// this field alone".
-type domainScreenPlan struct {
-	hashID   string
-	domain   string
-	workForm string
-}
-
-// MigrateDomainScreen rewrites `domain=film|tv` to `domain=screen` and
-// backfills `workForm` from `contentKind` across the whole corpus.
-//
-// Reads in two batched MGET passes (never a per-record SCAN — a single record
-// read costs a full keyspace sweep on this store), then writes only the records
-// that actually change.
-func (c *Client) MigrateDomainScreen(ctx context.Context) (int, error) {
-	plans, err := c.planDomainScreen(ctx)
-	if err != nil {
-		return 0, err
-	}
-	// Writes take the client lock themselves, so they happen after the read
-	// phase has released it.
-	fixed := 0
-	for _, p := range plans {
-		if p.domain != "" {
-			if err := c.SetProperty(p.hashID, "domain", p.domain); err != nil {
-				return fixed, fmt.Errorf("set domain on %s: %w", p.hashID, err)
-			}
-		}
-		if p.workForm != "" {
-			if err := c.SetProperty(p.hashID, "workForm", p.workForm); err != nil {
-				return fixed, fmt.Errorf("set workForm on %s: %w", p.hashID, err)
-			}
-		}
-		fixed++
-	}
-	return fixed, nil
-}
-
-// planDomainScreen is the read half: what would change, without changing it.
-func (c *Client) planDomainScreen(ctx context.Context) ([]domainScreenPlan, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
-	if err != nil {
-		return nil, fmt.Errorf("smembers index: %w", err)
-	}
-	if len(hashIDs) == 0 {
-		return nil, nil
-	}
-
-	// Three keys per record, in a fixed order so the MGET response decodes by
-	// position: keys[i*3+n] belongs to hashIDs[i].
-	const fieldsPerFile = 3
-	fields := [fieldsPerFile]string{"domain", "workForm", "contentKind"}
-	keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
-	for _, h := range hashIDs {
-		prefix := c.buildKeyPrefix(h)
-		for _, f := range fields {
-			keys = append(keys, prefix+f)
-		}
-	}
-
-	// Chunked so one MGET never carries the whole corpus.
-	const chunk = 3000
-	values := make([]interface{}, 0, len(keys))
-	for start := 0; start < len(keys); start += chunk {
-		end := start + chunk
-		if end > len(keys) {
-			end = len(keys)
-		}
-		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
-		if err != nil {
-			return nil, fmt.Errorf("mget: %w", err)
-		}
-		values = append(values, got...)
-	}
-
-	str := func(i int) string {
-		if i >= len(values) {
-			return ""
-		}
-		s, _ := values[i].(string)
-		return s
-	}
-
-	plans := make([]domainScreenPlan, 0)
-	for i, h := range hashIDs {
-		base := i * fieldsPerFile
-		domain, workForm, contentKind := str(base), str(base+1), str(base+2)
-
-		plan := domainScreenPlan{hashID: h}
-		if replacement, ok := legacyDomainReplacements[domain]; ok {
-			plan.domain = replacement
-		}
-		// A stored `workForm` is authoritative — a `pack` writer knew something
-		// the table cannot. Only an absent one is filled in.
-		if workForm == "" {
-			if form, ok := workFormByContentKind[contentKind]; ok {
-				plan.workForm = form
-			}
-		}
-		if plan.domain != "" || plan.workForm != "" {
-			plans = append(plans, plan)
-		}
-	}
-	return plans, nil
-}
-
-// MigrateDomainScreenWithTimeout is the convenience wrapper used by the API
-// handler. Generous timeout: the read phase is two batched passes, but the
-// write phase is one round-trip per changed field.
-func (c *Client) MigrateDomainScreenWithTimeout() (int, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-	return c.MigrateDomainScreen(ctx)
-}

internal/storage/domain_screen_migration_test.go

@@ -1,140 +0,0 @@
-package storage
-
-import (
-	"context"
-	"testing"
-)
-
-// seedRecord writes one record's classification fields straight through
-// SetProperty, which is what a real writer uses (it maintains the field index
-// and the root index the sweep reads from).
-func seedRecord(t *testing.T, c *Client, hashID string, fields map[string]string) {
-	t.Helper()
-	for k, v := range fields {
-		if err := c.SetProperty(hashID, k, v); err != nil {
-			t.Fatalf("seed %s/%s: %v", hashID, k, err)
-		}
-	}
-}
-
-// The merge (METADATA_KEYS.md §14.17) is paid in the data: `domain` is stored,
-// not derived on read, so a record left saying `film` is invisible to a
-// `domain:screen` wall. This is the sweep that pays it.
-func TestMigrateDomainScreen_RewritesLegacyDomainsAndBackfillsWorkForm(t *testing.T) {
-	c, _ := newTestClient(t, "")
-
-	seedRecord(t, c, "rec-movie", map[string]string{
-		"domain":      "film",
-		"contentKind": "movie",
-	})
-	seedRecord(t, c, "rec-episode", map[string]string{
-		"domain":      "tv",
-		"contentKind": "episode",
-	})
-	// Already current, and already carries both keys: must be left alone.
-	seedRecord(t, c, "rec-current", map[string]string{
-		"domain":      "screen",
-		"contentKind": "series",
-		"workForm":    "serial",
-	})
-	// A different domain entirely — the sweep must not touch it, but it is
-	// still owed the `workForm` backfill.
-	seedRecord(t, c, "rec-book", map[string]string{
-		"domain":      "literature",
-		"contentKind": "book",
-	})
-
-	fixed, err := c.MigrateDomainScreen(context.Background())
-	if err != nil {
-		t.Fatalf("migrate: %v", err)
-	}
-	if fixed != 3 {
-		t.Errorf("fixed = %d, want 3 (the two legacy rows + the book's backfill)", fixed)
-	}
-
-	for _, tc := range []struct {
-		hashID   string
-		domain   string
-		workForm string
-	}{
-		{"rec-movie", "screen", "standalone"},
-		{"rec-episode", "screen", "serial"},
-		{"rec-current", "screen", "serial"},
-		{"rec-book", "literature", "standalone"},
-	} {
-		m, err := c.GetMetadataFlat(tc.hashID)
-		if err != nil {
-			t.Fatalf("read %s: %v", tc.hashID, err)
-		}
-		if m["domain"] != tc.domain {
-			t.Errorf("%s domain = %q, want %q", tc.hashID, m["domain"], tc.domain)
-		}
-		if m["workForm"] != tc.workForm {
-			t.Errorf("%s workForm = %q, want %q", tc.hashID, m["workForm"], tc.workForm)
-		}
-	}
-}
-
-// A second pass must find nothing to do — the endpoint is called by hand on
-// each box, and "run it twice" must not mean "write the whole corpus twice".
-func TestMigrateDomainScreen_IsIdempotent(t *testing.T) {
-	c, _ := newTestClient(t, "mm:")
-
-	seedRecord(t, c, "rec-1", map[string]string{
-		"domain":      "tv",
-		"contentKind": "episode",
-	})
-
-	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
-		t.Fatalf("first pass: %v", err)
-	}
-	fixed, err := c.MigrateDomainScreen(context.Background())
-	if err != nil {
-		t.Fatalf("second pass: %v", err)
-	}
-	if fixed != 0 {
-		t.Errorf("second pass fixed = %d, want 0", fixed)
-	}
-}
-
-// `pack` is the one kind neither routing table can decide — a season pack is
-// `serial`, an album release `standalone`. A pack that reaches the sweep
-// without a `workForm` must keep none: inventing one writes a guess into the
-// record permanently.
-func TestMigrateDomainScreen_NeverGuessesAPacksWorkForm(t *testing.T) {
-	c, _ := newTestClient(t, "")
-
-	seedRecord(t, c, "rec-pack", map[string]string{
-		"domain":      "tv",
-		"contentKind": "pack",
-	})
-	// A writer that DID know keeps what it wrote.
-	seedRecord(t, c, "rec-album", map[string]string{
-		"domain":      "music",
-		"contentKind": "pack",
-		"workForm":    "standalone",
-	})
-
-	if _, err := c.MigrateDomainScreen(context.Background()); err != nil {
-		t.Fatalf("migrate: %v", err)
-	}
-
-	m, err := c.GetMetadataFlat("rec-pack")
-	if err != nil {
-		t.Fatalf("read: %v", err)
-	}
-	if m["domain"] != "screen" {
-		t.Errorf("domain = %q, want screen (the domain WAS knowable)", m["domain"])
-	}
-	if _, ok := m["workForm"]; ok {
-		t.Errorf("workForm = %q, want absent", m["workForm"])
-	}
-
-	a, err := c.GetMetadataFlat("rec-album")
-	if err != nil {
-		t.Fatalf("read: %v", err)
-	}
-	if a["workForm"] != "standalone" {
-		t.Errorf("workForm = %q, want standalone (writer-supplied, never overwritten)", a["workForm"])
-	}
-}

internal/storage/dual_root_migration.go

@@ -1,41 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-)
-
-// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
-// legacy storage model: /meta/{hash} write handlers that didn't resolve
-// CID → UUID first created a parallel midhash-rooted entry alongside the
-// watcher's UUID root.
-//
-// That pattern is now structurally impossible:
-//   - writes resolve via ResolveRoot before landing (see handlers), and
-//   - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
-//     UUID roots, with the reverse index cid:<cid> → uuid as the single
-//     resolution path.
-//
-// The legacy on-disk shape this migration targeted (midhash-rooted entries,
-// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
-// cut-over wipes Redis, so there is nothing to migrate. The function is kept
-// as a no-op so the admin endpoint and its wrappers continue to compile and
-// respond.
-func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
-	if c.client == nil {
-		return 0, fmt.Errorf("not connected")
-	}
-	return 0, nil
-}
-
-// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
-// handler and the startup hook.
-func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
-	return c.MigrateDualRoots(context.Background())
-}
-
-// MigrateDualRootsReport returns a human-readable summary. With the dual-root
-// pattern retired (see MigrateDualRoots), this always reports nothing to do.
-func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
-	return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
-}

internal/storage/field_index.go

@@ -1,486 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/redis/go-redis/v9"
-)
-
-// The per-record field-name index.
-//
-// A record is stored as one flat Redis STRING per leaf
-// (`file:<uuid>/<field>`), and that shape is deliberate — it is what lets
-// keyspace notifications fire per *field*, so meta-fuse / meta-editor /
-// meta-search can react to one field changing instead of diffing a whole hash
-// (see docs/project-architecture/persistent-storage.md, and
-// dev/test/suites/flat-keys.bats which asserts records are never hashes).
-//
-// The cost of that shape was that reading one record meant asking Redis "which
-// keys start with `file:<uuid>/`?", and Redis has no prefix index — `SCAN
-// MATCH` walks the ENTIRE keyspace. On a real store (445k keys, ~36 fields per
-// record) a single record read cost ~670 SCAN round-trips and ~0.5s of Redis
-// CPU; SCAN accounted for ~97.5% of all Redis CPU time on the box. That sat on
-// the byte path for every poster meta-share served, which is how it surfaced.
-//
-// So we keep the flat keys and index their *names*:
-//
-//	file:<uuid>/__fields__   SET{ "filePath", "sizeByte", "cids/bafk…", … }
-//
-// A read becomes SMEMBERS + a chunked MGET — two round-trips instead of ~670.
-// This mirrors `file:__index__` (the set of every root) one level down, and the
-// bulk reads already worked this way: `GetTuplesForAllFiles` composes its keys
-// from the index rather than scanning per root.
-//
-// **The set is derived, never authoritative.** The flat keys remain the source
-// of truth. A record whose set is missing (anything written before this landed)
-// falls back to the old SCAN and populates the set on the way out, so existing
-// deployments heal themselves on first touch with no migration and no
-// SchemaVersion bump — which matters, because a bump is a flag day that makes
-// operators wipe Redis (see schema_version.go).
-//
-// The consistency argument that makes a derived index safe here is meta-core's
-// single-writer rule: leader election publishes an empty `redisUrl` on purpose
-// (internal/leader/election.go), so nothing outside this process can write a
-// `file:` key behind the index's back.
-
-// FieldsField is the reserved field name holding a record's field-name index.
-//
-// It is a Redis SET living at `file:<uuid>/__fields__`, i.e. inside the same
-// prefix as the record's own fields. That keeps every existing sweep correct
-// for free — `DeleteMetadata`, `DeleteRoot` and `ClearAllMetadata` all
-// SCAN+DEL `file:<uuid>/*` and therefore drop the index along with the record.
-// The flip side is that every reader which enumerates that prefix has to skip
-// it, exactly as they already skip `file:__index__`.
-const FieldsField = "__fields__"
-
-// mgetBatch caps how many keys go into one MGET. Redis limits arguments per
-// command; 1000 is the batch size `GetTuplesForAllFiles` already uses.
-const mgetBatch = 1000
-
-// buildFieldsKey constructs the key for a record's field-name index set.
-func (c *Client) buildFieldsKey(hashID string) string {
-	return c.buildKeyPrefix(hashID) + FieldsField
-}
-
-// isReservedFieldName reports whether a field name is bookkeeping rather than
-// metadata, and so must never be returned to an API caller, exported into a
-// snapshot, or folded into the search haystack.
-func isReservedFieldName(field string) bool {
-	return field == FieldsField
-}
-
-// readMetadataLocked returns a record's full flat metadata. Caller must hold
-// c.mu (either mode).
-//
-// Fast path: SMEMBERS the field index, then MGET those keys. Fallback: the
-// legacy full-keyspace SCAN, whose result is used to populate the index so the
-// next read takes the fast path.
-//
-// Returns (nil, nil) for a record with no fields — callers treat that as 404.
-func (c *Client) readMetadataLocked(ctx context.Context, hashID string) (map[string]string, error) {
-	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return nil, fmt.Errorf("smembers fields: %w", err)
-	}
-
-	if len(fields) > 0 {
-		return c.mgetFieldsLocked(ctx, hashID, fields)
-	}
-
-	// No index yet: a record written before the index existed, or one that
-	// genuinely has no fields. Fall back to the scan, then backfill.
-	result, scanned, err := c.scanMetadataLocked(ctx, hashID)
-	if err != nil {
-		return nil, err
-	}
-	if len(scanned) > 0 {
-		// Best-effort: a failed backfill costs the next read a scan, it does
-		// not make the read wrong. Never fail the caller's read for it.
-		if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), toAny(scanned)...).Err(); err != nil {
-			logFieldIndexBackfillFailure(hashID, err)
-		}
-	}
-	if len(result) == 0 {
-		return nil, nil
-	}
-	return result, nil
-}
-
-// mgetFieldsLocked reads the named fields of a record in chunked MGETs.
-//
-// A field named in the index whose key is gone reads back as nil and is simply
-// skipped: a *stale* index entry is harmless. (A *missing* one would silently
-// drop data, which is why every write path maintains the set.)
-func (c *Client) mgetFieldsLocked(ctx context.Context, hashID string, fields []string) (map[string]string, error) {
-	prefix := c.buildKeyPrefix(hashID)
-
-	// Keep the reserved name out of the key list AND out of the result — it is
-	// a SET, so MGET would error on it in real Redis rather than return nil.
-	wanted := make([]string, 0, len(fields))
-	for _, f := range fields {
-		if !isReservedFieldName(f) {
-			wanted = append(wanted, f)
-		}
-	}
-	if len(wanted) == 0 {
-		return nil, nil
-	}
-
-	result := make(map[string]string, len(wanted))
-	for start := 0; start < len(wanted); start += mgetBatch {
-		end := start + mgetBatch
-		if end > len(wanted) {
-			end = len(wanted)
-		}
-		chunk := wanted[start:end]
-		keys := make([]string, len(chunk))
-		for i, f := range chunk {
-			keys[i] = prefix + f
-		}
-		values, err := c.client.MGet(ctx, keys...).Result()
-		if err != nil {
-			return nil, fmt.Errorf("mget failed: %w", err)
-		}
-		for i, v := range values {
-			if s, ok := v.(string); ok {
-				result[chunk[i]] = s
-			}
-		}
-	}
-
-	if len(result) == 0 {
-		return nil, nil
-	}
-	return result, nil
-}
-
-// scanMetadataLocked is the pre-index read: SCAN the record's key prefix, then
-// MGET. Retained as the fallback for un-indexed records and as the source for
-// the backfill. Returns the metadata plus the field names it saw (which
-// includes fields whose value came back nil, so the index stays complete).
-func (c *Client) scanMetadataLocked(ctx context.Context, hashID string) (map[string]string, []string, error) {
-	prefix := c.buildKeyPrefix(hashID)
-	result := make(map[string]string)
-	var fields []string
-
-	var cursor uint64
-	for {
-		keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, nil, fmt.Errorf("scan failed: %w", err)
-		}
-
-		if len(keys) > 0 {
-			wanted := make([]string, 0, len(keys))
-			for _, key := range keys {
-				field := strings.TrimPrefix(key, prefix)
-				if isReservedFieldName(field) {
-					continue
-				}
-				fields = append(fields, field)
-				wanted = append(wanted, key)
-			}
-			if len(wanted) > 0 {
-				values, err := c.client.MGet(ctx, wanted...).Result()
-				if err != nil {
-					return nil, nil, fmt.Errorf("mget failed: %w", err)
-				}
-				for i, key := range wanted {
-					if s, ok := values[i].(string); ok {
-						result[strings.TrimPrefix(key, prefix)] = s
-					}
-				}
-			}
-		}
-
-		cursor = nextCursor
-		if cursor == 0 {
-			break
-		}
-	}
-
-	return result, fields, nil
-}
-
-// addFieldsToPipe records field names in the index as part of an existing
-// pipeline, so a multi-field write stays one round-trip.
-func (c *Client) addFieldsToPipe(ctx context.Context, pipe redis.Pipeliner, hashID string, fields []string) {
-	names := filterReserved(fields)
-	if len(names) == 0 {
-		return
-	}
-	pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(names)...)
-}
-
-// addFieldLocked records a single field name in the index. Caller must hold
-// c.mu. Best-effort by the same argument as the backfill: the flat key is the
-// source of truth, and a lost index entry is repaired by the next scan
-// fallback — so it must not fail the write it accompanies.
-func (c *Client) addFieldLocked(ctx context.Context, hashID, field string) {
-	if isReservedFieldName(field) {
-		return
-	}
-	if err := c.client.SAdd(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
-		logFieldIndexWriteFailure(hashID, field, err)
-	}
-}
-
-// removeFieldLocked drops a field name from the index. Caller must hold c.mu.
-//
-// A stale entry left behind by a failure is harmless (the MGET reads nil and
-// skips it), so this is best-effort too.
-func (c *Client) removeFieldLocked(ctx context.Context, hashID, field string) {
-	if isReservedFieldName(field) {
-		return
-	}
-	if err := c.client.SRem(ctx, c.buildFieldsKey(hashID), field).Err(); err != nil {
-		logFieldIndexWriteFailure(hashID, field, err)
-	}
-}
-
-// fieldsWithPrefixLocked returns the record's indexed field names under a
-// field-name prefix (e.g. "cids/"), falling back to a SCAN when the record has
-// no index yet. Caller must hold c.mu.
-func (c *Client) fieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
-	fields, err := c.client.SMembers(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return nil, fmt.Errorf("smembers fields: %w", err)
-	}
-	if len(fields) == 0 {
-		// Un-indexed record: scan the narrower `<prefix><fieldPrefix>*` glob
-		// rather than the whole record, and do not backfill from it — a
-		// partial view must never be mistaken for a complete index.
-		return c.scanFieldsWithPrefixLocked(ctx, hashID, fieldPrefix)
-	}
-
-	var out []string
-	for _, f := range fields {
-		if isReservedFieldName(f) {
-			continue
-		}
-		if strings.HasPrefix(f, fieldPrefix) {
-			out = append(out, strings.TrimPrefix(f, fieldPrefix))
-		}
-	}
-	// SMEMBERS order is arbitrary and SCAN order was too, so the CID array in
-	// GetMetadataDocument used to shuffle between calls. Sorting costs nothing
-	// at these sizes and makes the response byte-stable, which matters for
-	// anything diffing or caching it.
-	sort.Strings(out)
-	return out, nil
-}
-
-func (c *Client) scanFieldsWithPrefixLocked(ctx context.Context, hashID, fieldPrefix string) ([]string, error) {
-	prefix := c.buildKeyPrefix(hashID) + fieldPrefix
-	var out []string
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan fields: %w", err)
-		}
-		for _, k := range keys {
-			out = append(out, strings.TrimPrefix(k, prefix))
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-	sort.Strings(out)
-	return out, nil
-}
-
-// WarmFieldIndexes builds every missing field index in a SINGLE keyspace pass.
-//
-// Without it the indexes still appear — each record's first read falls back to
-// a scan and backfills — but that transition is O(records x keyspace) and it
-// runs again from scratch after any wipe. Measured on the real store (12,345
-// records / 445k keys) the lazy path warmed at ~1.3 records/s: a ~2.6 HOUR tail
-// during which meta-core stays pinned exactly as it was before the fix. Worse,
-// `RepublishMetadata` walks every record at startup, so that tail is paid on
-// every restart.
-//
-// One bulk `file:*` scan grouped by record collapses it to a few seconds. This
-// is the same shape `GetSearchIndexTuples` already uses for the bulk read, and
-// for the same reason.
-//
-// Idempotent, and safe to run against a partially-indexed store: SADD of a
-// member that is already there is a no-op. Returns the number of records it
-// wrote an index for.
-func (c *Client) WarmFieldIndexes() (int, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return 0, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-
-	filePrefix := c.buildKey("file:") // {filePrefix}{hashID}/{field}
-	indexKey := c.buildIndexKey()
-
-	byRecord := make(map[string][]string)
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, filePrefix+"*", 1000).Result()
-		if err != nil {
-			return 0, fmt.Errorf("scan: %w", err)
-		}
-		for _, k := range keys {
-			if k == indexKey {
-				continue
-			}
-			// hashID has no '/'; field may contain them.
-			rest := strings.TrimPrefix(k, filePrefix)
-			slash := strings.IndexByte(rest, '/')
-			if slash < 0 {
-				continue
-			}
-			hashID, field := rest[:slash], rest[slash+1:]
-			if isReservedFieldName(field) {
-				// Already indexed — remember the record so we don't treat it
-				// as missing, but there is nothing to add for this key.
-				if _, seen := byRecord[hashID]; !seen {
-					byRecord[hashID] = nil
-				}
-				continue
-			}
-			byRecord[hashID] = append(byRecord[hashID], field)
-		}
-		cursor = next
-		if cursor == 0 {
-			break
-		}
-	}
-
-	written := 0
-	pipe := c.client.Pipeline()
-	pending := 0
-	for hashID, fields := range byRecord {
-		if len(fields) == 0 {
-			continue
-		}
-		pipe.SAdd(ctx, c.buildFieldsKey(hashID), toAny(fields)...)
-		written++
-		pending++
-		// Bound the pipeline so a large store doesn't buffer every command.
-		if pending >= 500 {
-			if _, err := pipe.Exec(ctx); err != nil {
-				return written, fmt.Errorf("sadd pipeline: %w", err)
-			}
-			pipe = c.client.Pipeline()
-			pending = 0
-		}
-	}
-	if pending > 0 {
-		if _, err := pipe.Exec(ctx); err != nil {
-			return written, fmt.Errorf("sadd pipeline: %w", err)
-		}
-	}
-	return written, nil
-}
-
-// recordHasFieldsLocked reports whether a record still has at least one
-// property key. Caller must hold c.mu.
-//
-// `__fields__` must not count — it lives under the record's own prefix, so a
-// naive "does any `file:<id>/*` key exist?" probe would answer yes for a record
-// whose every property has been deleted, and the root could never be
-// de-indexed.
-func (c *Client) recordHasFieldsLocked(ctx context.Context, hashID string) (bool, error) {
-	n, err := c.client.SCard(ctx, c.buildFieldsKey(hashID)).Result()
-	if err != nil && err != redis.Nil {
-		return false, fmt.Errorf("scard fields: %w", err)
-	}
-	if n > 0 {
-		return true, nil
-	}
-
-	// Either genuinely empty, or an un-indexed record from before the index
-	// existed — the scan is the only way to tell those apart.
-	prefix := c.buildKeyPrefix(hashID)
-	var cursor uint64
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
-		if err != nil {
-			return false, fmt.Errorf("scan failed: %w", err)
-		}
-		for _, k := range keys {
-			if !isReservedFieldName(strings.TrimPrefix(k, prefix)) {
-				return true, nil
-			}
-		}
-		cursor = next
-		if cursor == 0 {
-			return false, nil
-		}
-	}
-}
-
-// splitRecordKey decomposes a full Redis key into (hashID, field) when it is a
-// record property key. Reports ok=false for anything else — the index set
-// itself, `file:__index__`, and any key outside the `file:` namespace.
-//
-// This is what lets the raw KV escape hatch (SetRawValue / DeleteRawKey) keep
-// the index honest without its callers knowing the layout.
-func (c *Client) splitRecordKey(key string) (hashID, field string, ok bool) {
-	rest := key
-	if c.prefix != "" {
-		var found bool
-		rest, found = strings.CutPrefix(rest, c.prefix)
-		if !found {
-			return "", "", false
-		}
-	}
-	rest, found := strings.CutPrefix(rest, "file:")
-	if !found {
-		return "", "", false
-	}
-	hashID, field, found = strings.Cut(rest, "/")
-	if !found || hashID == "" || field == "" {
-		// `file:__index__` has no slash and lands here.
-		return "", "", false
-	}
-	if isReservedFieldName(field) {
-		return "", "", false
-	}
-	return hashID, field, true
-}
-
-// The index is derived and self-healing, so a write failure degrades
-// performance (the next read falls back to a scan and re-populates) rather
-// than correctness. Log it rather than failing the caller's write — but log it
-// loudly, because a *persistent* failure means every read is scanning again.
-func logFieldIndexWriteFailure(hashID, field string, err error) {
-	log.Printf("[Storage] field index update failed for %s/%s: %v (reads fall back to SCAN)", hashID, field, err)
-}
-
-func logFieldIndexBackfillFailure(hashID string, err error) {
-	log.Printf("[Storage] field index backfill failed for %s: %v (reads keep falling back to SCAN)", hashID, err)
-}
-
-func filterReserved(fields []string) []string {
-	out := make([]string, 0, len(fields))
-	for _, f := range fields {
-		if !isReservedFieldName(f) {
-			out = append(out, f)
-		}
-	}
-	return out
-}
-
-func toAny(ss []string) []interface{} {
-	out := make([]interface{}, len(ss))
-	for i, s := range ss {
-		out[i] = s
-	}
-	return out
-}

internal/storage/field_index_test.go

@@ -1,464 +0,0 @@
-package storage
-
-import (
-	"context"
-	"reflect"
-	"sort"
-	"testing"
-)
-
-// Tests for the per-record field-name index (field_index.go).
-//
-// The index is what makes a record read O(fields) instead of O(keyspace). It is
-// derived state, so the properties worth pinning are: (1) a read served from
-// the index returns exactly what a scan would have, (2) a record with no index
-// still reads correctly and gets one, and (3) every write path keeps it in
-// sync — because a *missing* entry silently drops a field from every
-// subsequent read, which is the one failure mode that would be invisible.
-//
-// Every case runs with and without a namespace prefix, matching clear_test.go.
-
-func forEachPrefix(t *testing.T, fn func(t *testing.T, c *Client)) {
-	t.Helper()
-	for _, prefix := range []string{"", "mm:"} {
-		name := "no-prefix"
-		if prefix != "" {
-			name = "prefix-" + prefix
-		}
-		t.Run(name, func(t *testing.T) {
-			c, _ := newTestClient(t, prefix)
-			fn(t, c)
-		})
-	}
-}
-
-func fieldNames(t *testing.T, c *Client, hashID string) []string {
-	t.Helper()
-	got, err := c.client.SMembers(context.Background(), c.buildFieldsKey(hashID)).Result()
-	if err != nil {
-		t.Fatalf("smembers fields: %v", err)
-	}
-	sort.Strings(got)
-	return got
-}
-
-func mustGetMetadata(t *testing.T, c *Client, hashID string) map[string]string {
-	t.Helper()
-	got, err := c.GetMetadataFlat(hashID)
-	if err != nil {
-		t.Fatalf("GetMetadataFlat: %v", err)
-	}
-	return got
-}
-
-// SetMetadataFlat must index every field it writes, and the read must come back
-// through the index rather than a scan.
-func TestFieldIndex_SetMetadataFlatIndexesFields(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		want := map[string]string{
-			"filePath":                      "/files/watch/a.mkv",
-			"sizeByte":                      "123",
-			"cids/bafkreiaaa":               "true",
-			"titles/jpn/ナルト":                "true",
-			"categories/TV/UHD":             "true",
-			"info/files/0/path":             "a.mkv",
-			"provenance":                    `{"source":"gateway"}`,
-			"plot/eng":                      "a plot with / slashes",
-			"stream/3":                      "eng",
-			"genres/action":                 "true",
-			"originaltitle":                 "Naruto",
-			"deeply/nested/field/name/here": "x",
-		}
-		if err := c.SetMetadataFlat("root1", want); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		if got := mustGetMetadata(t, c, "root1"); !reflect.DeepEqual(got, want) {
-			t.Errorf("GetMetadataFlat mismatch\n got %v\nwant %v", got, want)
-		}
-
-		// The index must name every field — including the ones containing '/',
-		// which is where a naive "split on the first slash" would go wrong.
-		wantNames := make([]string, 0, len(want))
-		for k := range want {
-			wantNames = append(wantNames, k)
-		}
-		sort.Strings(wantNames)
-		if got := fieldNames(t, c, "root1"); !reflect.DeepEqual(got, wantNames) {
-			t.Errorf("field index mismatch\n got %v\nwant %v", got, wantNames)
-		}
-	})
-}
-
-// A record written before the index existed has flat keys but no set. The read
-// must still be correct, and must leave an index behind so the next read takes
-// the fast path. This is the migration path for the ~12k records already on
-// disk: no flag day, no SchemaVersion bump.
-func TestFieldIndex_LazyBackfillForUnindexedRecord(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		prefix := c.buildKeyPrefix("legacy")
-		want := map[string]string{
-			"filePath":        "/files/watch/old.mkv",
-			"sizeByte":        "42",
-			"cids/bafkreiold": "true",
-		}
-		// Write the flat keys directly, bypassing every index-maintaining path
-		// — exactly the shape a pre-upgrade store is in.
-		for f, v := range want {
-			if err := c.client.Set(ctx, prefix+f, v, 0).Err(); err != nil {
-				t.Fatalf("seed %s: %v", f, err)
-			}
-		}
-		if names := fieldNames(t, c, "legacy"); len(names) != 0 {
-			t.Fatalf("precondition: expected no index, got %v", names)
-		}
-
-		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
-			t.Errorf("scan-fallback read mismatch\n got %v\nwant %v", got, want)
-		}
-
-		wantNames := []string{"cids/bafkreiold", "filePath", "sizeByte"}
-		if got := fieldNames(t, c, "legacy"); !reflect.DeepEqual(got, wantNames) {
-			t.Errorf("backfill mismatch\n got %v\nwant %v", got, wantNames)
-		}
-
-		// Second read, now served from the index, must agree with the first.
-		if got := mustGetMetadata(t, c, "legacy"); !reflect.DeepEqual(got, want) {
-			t.Errorf("indexed read disagrees with scan read\n got %v\nwant %v", got, want)
-		}
-	})
-}
-
-// Each single-field write path must add to the index, and each delete path must
-// remove from it. A field written but not indexed vanishes from every later
-// read — silently — so this is the property that matters most.
-func TestFieldIndex_SingleFieldWritePathsStayInSync(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetProperty("root2", "title", "Naruto"); err != nil {
-			t.Fatalf("SetProperty: %v", err)
-		}
-		if _, err := c.AddToSet("root2", "tags", "anime"); err != nil {
-			t.Fatalf("AddToSet: %v", err)
-		}
-		if _, err := c.MergeMetadataFlat("root2", map[string]string{"year": "2002"}); err != nil {
-			t.Fatalf("MergeMetadataFlat: %v", err)
-		}
-
-		want := map[string]string{"title": "Naruto", "tags": "anime", "year": "2002"}
-		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
-			t.Errorf("after writes\n got %v\nwant %v", got, want)
-		}
-
-		// DeleteProperty drops the key and the index entry.
-		if err := c.DeleteProperty("root2", "title"); err != nil {
-			t.Fatalf("DeleteProperty: %v", err)
-		}
-		// RemoveFromSet deletes the key once its last member is gone.
-		if _, err := c.RemoveFromSet("root2", "tags", "anime"); err != nil {
-			t.Fatalf("RemoveFromSet: %v", err)
-		}
-
-		want = map[string]string{"year": "2002"}
-		if got := mustGetMetadata(t, c, "root2"); !reflect.DeepEqual(got, want) {
-			t.Errorf("after deletes\n got %v\nwant %v", got, want)
-		}
-		if got := fieldNames(t, c, "root2"); !reflect.DeepEqual(got, []string{"year"}) {
-			t.Errorf("index after deletes = %v, want [year]", got)
-		}
-	})
-}
-
-// The raw KV editor can mint or drop a record property out of band. It is the
-// one write path whose caller does not know the record layout, so the storage
-// layer has to keep the index honest for it.
-func TestFieldIndex_RawKeyEditorStaysInSync(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetRawValue("file:root3/hand-written", "v"); err != nil {
-			t.Fatalf("SetRawValue: %v", err)
-		}
-		if got := fieldNames(t, c, "root3"); !reflect.DeepEqual(got, []string{"hand-written"}) {
-			t.Errorf("index after SetRawValue = %v", got)
-		}
-		if got := mustGetMetadata(t, c, "root3"); !reflect.DeepEqual(got, map[string]string{"hand-written": "v"}) {
-			t.Errorf("read after SetRawValue = %v", got)
-		}
-
-		if _, err := c.DeleteRawKey("file:root3/hand-written"); err != nil {
-			t.Fatalf("DeleteRawKey: %v", err)
-		}
-		if got := fieldNames(t, c, "root3"); len(got) != 0 {
-			t.Errorf("index after DeleteRawKey = %v, want empty", got)
-		}
-
-		// Keys outside the record namespace must be left alone entirely.
-		if err := c.SetRawValue("cid:bafkreiaaa", "root3"); err != nil {
-			t.Fatalf("SetRawValue non-record: %v", err)
-		}
-		if _, _, ok := c.splitRecordKey(c.buildKey("cid:bafkreiaaa")); ok {
-			t.Error("cid: key was treated as a record property")
-		}
-		if _, _, ok := c.splitRecordKey(c.buildIndexKey()); ok {
-			t.Error("file:__index__ was treated as a record property")
-		}
-	})
-}
-
-// Mint and AddAlias write fields through their own pipelines, so they need the
-// index too. AddAlias in particular is how `cids/<cid>` members appear — miss
-// it and a record's CID key-set becomes invisible to every read.
-func TestFieldIndex_MintAndAliasIndexTheirFields(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		uuid, err := c.Mint("/files/watch/x.mkv", 10, 20)
-		if err != nil {
-			t.Fatalf("Mint: %v", err)
-		}
-		if got := fieldNames(t, c, uuid); !reflect.DeepEqual(got, []string{"filePath", "mtimeNano", "sizeByte"}) {
-			t.Errorf("index after Mint = %v", got)
-		}
-
-		if err := c.AddAlias(uuid, "bafkreiaaa"); err != nil {
-			t.Fatalf("AddAlias: %v", err)
-		}
-		got := mustGetMetadata(t, c, uuid)
-		if got["cids/bafkreiaaa"] != "true" {
-			t.Errorf("alias key-set member missing from read: %v", got)
-		}
-
-		// And the CID enumeration, which now reads the index instead of
-		// scanning, must see it.
-		cids, err := c.cidsForRootLocked(context.Background(), uuid)
-		if err != nil {
-			t.Fatalf("cidsForRootLocked: %v", err)
-		}
-		if !reflect.DeepEqual(cids, []string{"bafkreiaaa"}) {
-			t.Errorf("cidsForRoot = %v, want [bafkreiaaa]", cids)
-		}
-	})
-}
-
-// The index lives under the record's own prefix, so the "is this record empty?"
-// probe would otherwise always find it and the root could never be de-indexed.
-func TestFieldIndex_IndexRemoveIfEmptyIgnoresTheIndexItself(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetProperty("root4", "title", "x"); err != nil {
-			t.Fatalf("SetProperty: %v", err)
-		}
-
-		removed, err := c.IndexRemoveIfEmpty("root4")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if removed {
-			t.Error("de-indexed a record that still has a field")
-		}
-
-		if err := c.DeleteProperty("root4", "title"); err != nil {
-			t.Fatalf("DeleteProperty: %v", err)
-		}
-		removed, err = c.IndexRemoveIfEmpty("root4")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if !removed {
-			t.Fatal("record with no fields left was not de-indexed — the field index masked emptiness")
-		}
-		// ...and the now-empty index key must not be left orphaned.
-		n, err := c.client.Exists(context.Background(), c.buildFieldsKey("root4")).Result()
-		if err != nil {
-			t.Fatalf("exists: %v", err)
-		}
-		if n != 0 {
-			t.Error("orphaned field index left behind after de-indexing")
-		}
-	})
-}
-
-// The same probe must still work for a record that predates the index.
-func TestFieldIndex_IndexRemoveIfEmptyHandlesUnindexedRecord(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		if err := c.client.Set(ctx, c.buildKeyPrefix("legacy2")+"title", "x", 0).Err(); err != nil {
-			t.Fatalf("seed: %v", err)
-		}
-		if err := c.IndexAdd("legacy2"); err != nil {
-			t.Fatalf("IndexAdd: %v", err)
-		}
-		removed, err := c.IndexRemoveIfEmpty("legacy2")
-		if err != nil {
-			t.Fatalf("IndexRemoveIfEmpty: %v", err)
-		}
-		if removed {
-			t.Error("de-indexed an un-indexed record that still has a field")
-		}
-	})
-}
-
-// The reserved name is bookkeeping. It must never reach an API caller as a
-// metadata field, and an inbound payload must never be able to overwrite the
-// SET with a string — that would break every subsequent read of the record.
-func TestFieldIndex_ReservedNameIsNeverMetadata(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		err := c.SetMetadataFlat("root5", map[string]string{
-			"title":     "x",
-			FieldsField: "malicious",
-		})
-		if err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		got := mustGetMetadata(t, c, "root5")
-		if _, present := got[FieldsField]; present {
-			t.Errorf("%s surfaced as a metadata field: %v", FieldsField, got)
-		}
-		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
-			t.Errorf("read = %v, want just title", got)
-		}
-
-		// The index must still be a SET — i.e. not clobbered by the payload.
-		if got := fieldNames(t, c, "root5"); !reflect.DeepEqual(got, []string{"title"}) {
-			t.Errorf("index = %v, want [title]", got)
-		}
-
-		if err := c.SetProperty("root5", FieldsField, "malicious"); err == nil {
-			t.Error("SetProperty accepted the reserved field name")
-		}
-	})
-}
-
-// Deleting a record must take its index with it, or a recreated root would
-// inherit phantom field names.
-func TestFieldIndex_DeleteMetadataRemovesTheIndex(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetMetadataFlat("root6", map[string]string{"title": "x", "year": "2002"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-		if _, err := c.DeleteMetadata("root6"); err != nil {
-			t.Fatalf("DeleteMetadata: %v", err)
-		}
-		if got := fieldNames(t, c, "root6"); len(got) != 0 {
-			t.Errorf("index survived DeleteMetadata: %v", got)
-		}
-		got, err := c.GetMetadataFlat("root6")
-		if err != nil {
-			t.Fatalf("GetMetadataFlat: %v", err)
-		}
-		if got != nil {
-			t.Errorf("deleted record still reads back: %v", got)
-		}
-	})
-}
-
-// A stale index entry (name present, key gone) is the benign direction: the
-// MGET reads nil and the field is skipped rather than erroring.
-func TestFieldIndex_StaleEntryIsHarmless(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-		if err := c.SetMetadataFlat("root7", map[string]string{"title": "x"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-		// Name a field that has no key behind it.
-		if err := c.client.SAdd(ctx, c.buildFieldsKey("root7"), "ghost").Err(); err != nil {
-			t.Fatalf("sadd ghost: %v", err)
-		}
-
-		got := mustGetMetadata(t, c, "root7")
-		if !reflect.DeepEqual(got, map[string]string{"title": "x"}) {
-			t.Errorf("read = %v, want just title", got)
-		}
-	})
-}
-
-// WarmFieldIndexes is what keeps an upgrade from spending hours reading
-// un-indexed records one full-keyspace scan at a time. It must index every
-// record in one pass, be idempotent, and leave already-indexed records alone.
-func TestFieldIndex_WarmBuildsEveryMissingIndex(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		ctx := context.Background()
-
-		// Two legacy records seeded straight into Redis (no index), plus one
-		// written through the normal path (already indexed).
-		legacy := map[string]map[string]string{
-			"old1": {"filePath": "/a.mkv", "cids/bafkreiaaa": "true"},
-			"old2": {"filePath": "/b.mkv", "titles/jpn/ナルト": "true", "deep/a/b": "x"},
-		}
-		for root, fields := range legacy {
-			for f, v := range fields {
-				if err := c.client.Set(ctx, c.buildKeyPrefix(root)+f, v, 0).Err(); err != nil {
-					t.Fatalf("seed: %v", err)
-				}
-			}
-			if err := c.IndexAdd(root); err != nil {
-				t.Fatalf("IndexAdd: %v", err)
-			}
-		}
-		if err := c.SetMetadataFlat("fresh", map[string]string{"title": "x"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		n, err := c.WarmFieldIndexes()
-		if err != nil {
-			t.Fatalf("WarmFieldIndexes: %v", err)
-		}
-		if n < 2 {
-			t.Errorf("warmed %d records, want at least the 2 legacy ones", n)
-		}
-
-		if got := fieldNames(t, c, "old1"); !reflect.DeepEqual(got, []string{"cids/bafkreiaaa", "filePath"}) {
-			t.Errorf("old1 index = %v", got)
-		}
-		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
-			t.Errorf("old2 index = %v", got)
-		}
-		if got := fieldNames(t, c, "fresh"); !reflect.DeepEqual(got, []string{"title"}) {
-			t.Errorf("fresh index disturbed = %v", got)
-		}
-
-		// Reads must now agree with what the records actually hold.
-		for root, want := range legacy {
-			if got := mustGetMetadata(t, c, root); !reflect.DeepEqual(got, want) {
-				t.Errorf("%s read = %v, want %v", root, got, want)
-			}
-		}
-
-		// Idempotent: a second pass changes nothing.
-		if _, err := c.WarmFieldIndexes(); err != nil {
-			t.Fatalf("second WarmFieldIndexes: %v", err)
-		}
-		if got := fieldNames(t, c, "old2"); !reflect.DeepEqual(got, []string{"deep/a/b", "filePath", "titles/jpn/ナルト"}) {
-			t.Errorf("old2 index after second warm = %v", got)
-		}
-	})
-}
-
-// The bulk search read enumerates the whole file: keyspace and MGETs what it
-// finds. The index keys are SETs living in that same namespace, so it has to
-// skip them or the MGET fails with WRONGTYPE and search dies wholesale.
-func TestFieldIndex_SearchIndexTuplesSkipTheIndexKeys(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		if err := c.SetMetadataFlat("root8", map[string]string{
-			"title":    "Naruto Shippuden",
-			"fileName": "naruto.mkv",
-		}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		tuples, err := c.GetSearchIndexTuples()
-		if err != nil {
-			t.Fatalf("GetSearchIndexTuples: %v", err)
-		}
-		var found bool
-		for _, tu := range tuples {
-			if tu.HashID != "root8" {
-				continue
-			}
-			found = true
-			if _, present := tu.Fields[FieldsField]; present {
-				t.Errorf("%s leaked into the search tuple: %v", FieldsField, tu.Fields)
-			}
-		}
-		if !found {
-			t.Fatal("record missing from search tuples")
-		}
-	})
-}

internal/storage/literature_echo_sweep.go

@@ -1,326 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/redis/go-redis/v9"
-
-	"github.com/metazla/meta-core/internal/cid"
-)
-
-// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
-// identity and position keys are resolved, never copied from a query, and a
-// card carries no position keys and no `workcid`.
-//
-// # Why a sweep
-//
-// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
-// own filter values onto what it returned. The gateway persists whatever passes
-// the filter, so the copy became a permanent binding — and stored records never
-// pick up the writer fix: a covered `(upstream, query)` never reaches the
-// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
-// one. So the copies already at rest are removed here, once.
-//
-// What the copies cost: an `anilistid` on a MangaDex site card claims an
-// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
-// anchor — the site card then competes with (and can hide) the real AniList
-// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
-// join and sort like a position inside one.
-//
-// # What is swept, and what deliberately is not
-//
-//   - Only `domain=literature` + `fileType=card` records. Nothing else is read
-//     for writing, whatever keys it carries.
-//   - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
-//     go from **every** such card: a card is a work, never a position in one,
-//     so no legitimate writer puts them there.
-//   - `anilistid` goes only from a **site card** — a card whose own `cids/`
-//     key-set holds a site-form card locator and no locator of any other form.
-//     On an anchor card the id *is* the card's identity (its locator is
-//     `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
-//     it: which half the id belongs to is not recoverable at rest, and leaving
-//     a stamp is the safe direction.
-//   - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
-//     Matched by rule, and a query-copied stamp is indistinguishable at rest
-//     from a title-matched one — deleting would drop real bindings with the
-//     fake ones.
-//
-// # Contract
-//
-// Dry-run unless `apply`: the sweep deletes keys, so the default call only
-// reports. Idempotent: a swept record plans nothing on the next pass, so the
-// endpoint is safe to call repeatedly and on a fresh box.
-
-// literatureEchoAnchorSources are the card-locator sources that name an id-bag
-// work (an anchor), never a site.
-//
-// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
-// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
-// excludes them and the two rules agree on every locator minted today. They are
-// listed so the rule reads on its own here.
-var literatureEchoAnchorSources = map[string]bool{
-	"anilist":     true,
-	"openlibrary": true,
-	"tmdb":        true,
-	"imdb":        true,
-	"musicbrainz": true,
-	"mal":         true,
-}
-
-// literatureCardEchoFields are removed from every literature card carrying
-// them — position keys and the work reference, none of which a card may hold.
-var literatureCardEchoFields = []string{
-	"workcid",
-	"chapterNumber",
-	"volumeNumber",
-	"chapterStart",
-	"chapterEnd",
-}
-
-// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
-// then `anilistid`, then the echo fields in literatureCardEchoFields order.
-var literatureEchoReadFields = append(
-	[]string{"domain", "fileType", "anilistid"},
-	literatureCardEchoFields...,
-)
-
-// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
-// records; ByKey counts fields per key — removed on an apply pass, that would
-// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
-type LiteratureEchoSweep struct {
-	Planned int
-	Fixed   int
-	ByKey   map[string]int
-}
-
-// literatureEchoPlan is one record's pending deletes, in a fixed order
-// (`anilistid` first, then literatureCardEchoFields order).
-type literatureEchoPlan struct {
-	hashID string
-	fields []string
-}
-
-// isSiteFormCardSource is the site-form rule on an already-decoded source: not
-// an anchor source, and shaped like a host.
-func isSiteFormCardSource(source string) bool {
-	return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
-}
-
-// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
-// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
-func isSiteFormCardLocator(cidStr string) bool {
-	source, _, err := cid.DecodeCard(cidStr)
-	return err == nil && isSiteFormCardSource(source)
-}
-
-// isSiteCard reports whether a card's field names prove it a site card: at
-// least one site-form `cids/` card locator and no card locator of another form.
-// Non-card members (a content CID) say nothing about which work the card is and
-// are ignored. An empty field list — a record with no field-name index — is
-// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
-func isSiteCard(fields []string) bool {
-	site := false
-	for _, f := range fields {
-		member := cidFromKeysetField(f)
-		if member == "" {
-			continue
-		}
-		source, _, err := cid.DecodeCard(member)
-		if err != nil {
-			continue
-		}
-		if !isSiteFormCardSource(source) {
-			return false
-		}
-		site = true
-	}
-	return site
-}
-
-// SweepLiteratureEchoes removes query echoes stored on literature cards (see
-// the file comment). With apply=false it only plans.
-//
-// Reads in batched passes (never a per-record SCAN — a single record read
-// costs a full keyspace sweep on this store), then deletes only the planned
-// fields, through DeleteProperty so the field-name index and the search-dirty
-// set stay consistent.
-func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
-	res := LiteratureEchoSweep{ByKey: map[string]int{}}
-
-	plans, err := c.planLiteratureEchoes(ctx)
-	if err != nil {
-		return res, err
-	}
-	res.Planned = len(plans)
-
-	if !apply {
-		for _, p := range plans {
-			for _, f := range p.fields {
-				res.ByKey[f]++
-			}
-		}
-		return res, nil
-	}
-
-	// Writes take the client lock themselves, so they happen after the read
-	// phase has released it. ByKey counts what was actually deleted, so a
-	// failure midway reports the true partial state.
-	for _, p := range plans {
-		// DeleteProperty carries its own per-call timeout; checking ctx here is
-		// what makes the wrapper's deadline bound the whole write phase.
-		if err := ctx.Err(); err != nil {
-			return res, err
-		}
-		for _, f := range p.fields {
-			if err := c.DeleteProperty(p.hashID, f); err != nil {
-				return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
-			}
-			res.ByKey[f]++
-		}
-		res.Fixed++
-	}
-	return res, nil
-}
-
-// planLiteratureEchoes is the read half: what would be deleted, without
-// deleting it.
-func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
-	if err != nil {
-		return nil, fmt.Errorf("smembers index: %w", err)
-	}
-	if len(hashIDs) == 0 {
-		return nil, nil
-	}
-
-	// Pass 1: the gate and echo fields for every record, in a fixed order so
-	// the MGET response decodes by position: keys[i*per+n] belongs to
-	// hashIDs[i].
-	per := len(literatureEchoReadFields)
-	keys := make([]string, 0, len(hashIDs)*per)
-	for _, h := range hashIDs {
-		prefix := c.buildKeyPrefix(h)
-		for _, f := range literatureEchoReadFields {
-			keys = append(keys, prefix+f)
-		}
-	}
-
-	// Chunked so one MGET never carries the whole corpus.
-	const chunk = 3000
-	values := make([]interface{}, 0, len(keys))
-	for start := 0; start < len(keys); start += chunk {
-		end := start + chunk
-		if end > len(keys) {
-			end = len(keys)
-		}
-		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
-		if err != nil {
-			return nil, fmt.Errorf("mget: %w", err)
-		}
-		values = append(values, got...)
-	}
-
-	// A nil slot is an absent key. A present key holding "" is still an echo
-	// and still goes, so presence — not the value — decides.
-	get := func(i int) (string, bool) {
-		if i >= len(values) || values[i] == nil {
-			return "", false
-		}
-		s, ok := values[i].(string)
-		return s, ok
-	}
-
-	type candidate struct {
-		hashID     string
-		hasAnilist bool
-		echoes     []string
-	}
-	var cands []candidate
-	var probe []string // candidates whose anilistid needs their cids/ key-set
-	for i, h := range hashIDs {
-		base := i * per
-		domain, _ := get(base)
-		fileType, _ := get(base + 1)
-		if domain != "literature" || fileType != "card" {
-			continue
-		}
-		cand := candidate{hashID: h}
-		_, cand.hasAnilist = get(base + 2)
-		for j, f := range literatureCardEchoFields {
-			if _, ok := get(base + 3 + j); ok {
-				cand.echoes = append(cand.echoes, f)
-			}
-		}
-		if !cand.hasAnilist && len(cand.echoes) == 0 {
-			continue
-		}
-		if cand.hasAnilist {
-			probe = append(probe, h)
-		}
-		cands = append(cands, cand)
-	}
-
-	// Pass 2: the field-name index of only the cards carrying an `anilistid`
-	// — the `cids/` members are field NAMES, so the index is where they are
-	// listed. Pipelined per chunk; still no SCAN.
-	//
-	// A card with no index (never touched since before the index existed, and
-	// missed by the startup WarmFieldIndexes pass) reads as an empty set and
-	// keeps its `anilistid`: proving it a site card would need exactly the
-	// per-record SCAN this sweep refuses, and a stamp left in place is the
-	// safe direction.
-	siteCard := make(map[string]bool, len(probe))
-	for start := 0; start < len(probe); start += mgetBatch {
-		end := start + mgetBatch
-		if end > len(probe) {
-			end = len(probe)
-		}
-		pipe := c.client.Pipeline()
-		cmds := make([]*redis.StringSliceCmd, 0, end-start)
-		for _, h := range probe[start:end] {
-			cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
-		}
-		if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
-			return nil, fmt.Errorf("smembers fields: %w", err)
-		}
-		for k, cmd := range cmds {
-			fields, err := cmd.Result()
-			if err != nil && err != redis.Nil {
-				return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
-			}
-			siteCard[probe[start+k]] = isSiteCard(fields)
-		}
-	}
-
-	plans := make([]literatureEchoPlan, 0, len(cands))
-	for _, cand := range cands {
-		var fields []string
-		if cand.hasAnilist && siteCard[cand.hashID] {
-			fields = append(fields, "anilistid")
-		}
-		fields = append(fields, cand.echoes...)
-		if len(fields) > 0 {
-			plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
-		}
-	}
-	return plans, nil
-}
-
-// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
-// handler. Generous timeout: the read phase is batched, but the write phase is
-// one round-trip per deleted field.
-func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-	return c.SweepLiteratureEchoes(ctx, apply)
-}

internal/storage/literature_echo_sweep_test.go

@@ -1,249 +0,0 @@
-package storage
-
-import (
-	"context"
-	"encoding/base32"
-	"encoding/binary"
-	"reflect"
-	"strings"
-	"testing"
-)
-
-// Pinned from meta-read `src/card_cid.rs` — literals, not round-trips.
-const (
-	literatureAnchorCardCID = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy" // ("anilist", "manga:105778")
-	literatureSiteCardCID   = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
-	// A non-card CID a card may also carry.
-	literatureContentCID = "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
-)
-
-// mintCardCID encodes a card locator the way meta-feeder-sdk
-// `compute_card_cid` does. Asserted against the pinned literals before use.
-func mintCardCID(source, id string) string {
-	digest := binary.AppendUvarint(nil, uint64(len(source)))
-	digest = append(digest, source...)
-	digest = append(digest, id...)
-	wire := []byte{0x01}
-	wire = binary.AppendUvarint(wire, 0x1007)
-	wire = append(wire, 0x00)
-	wire = binary.AppendUvarint(wire, uint64(len(digest)))
-	wire = append(wire, digest...)
-	return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(wire))
-}
-
-func TestIsSiteFormCardLocator(t *testing.T) {
-	if got := mintCardCID("anilist", "manga:105778"); got != literatureAnchorCardCID {
-		t.Fatalf("test encoder drifted: %s != %s", got, literatureAnchorCardCID)
-	}
-	if got := mintCardCID("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"); got != literatureSiteCardCID {
-		t.Fatalf("test encoder drifted: %s != %s", got, literatureSiteCardCID)
-	}
-
-	for _, tc := range []struct {
-		name string
-		cid  string
-		want bool
-	}{
-		{"anilist anchor (golden)", literatureAnchorCardCID, false},
-		{"mangadex site (golden)", literatureSiteCardCID, true},
-		{"openlibrary anchor", mintCardCID("openlibrary", "OL45804W"), false},
-		{"another site", mintCardCID("weebcentral.com", "/series/01J76XY/chainsaw-man"), true},
-		// Not an anchor source, but not host-shaped either: meta-read reads it
-		// as an id-bag key, so it must not be treated as a site.
-		{"dotless unknown source", mintCardCID("suwayomi", "42"), false},
-		{"content CID", literatureContentCID, false},
-		{"title key", "title:chainsaw man", false},
-	} {
-		if got := isSiteFormCardLocator(tc.cid); got != tc.want {
-			t.Errorf("%s: isSiteFormCardLocator = %v, want %v", tc.name, got, tc.want)
-		}
-	}
-}
-
-// seedLiteratureCorpus writes one record per rule the sweep has to honour.
-func seedLiteratureCorpus(t *testing.T, c *Client) {
-	t.Helper()
-
-	// A site card carrying every echo: all of it goes.
-	seedRecord(t, c, "site-card", map[string]string{
-		"domain":                        "literature",
-		"fileType":                      "card",
-		"title":                         "Chainsaw Man",
-		"cids/" + literatureSiteCardCID: "true",
-		"anilistid":                     "105778",
-		"workcid":                       literatureSiteCardCID,
-		"chapterNumber":                 "12",
-		"volumeNumber":                  "2",
-		"chapterStart":                  "1",
-		"chapterEnd":                    "20",
-	})
-	// An anchor card: its anilistid is its identity and stays; the position
-	// echo and the workcid still go.
-	seedRecord(t, c, "anchor-card", map[string]string{
-		"domain":                          "literature",
-		"fileType":                        "card",
-		"title":                           "Chainsaw Man",
-		"cids/" + literatureAnchorCardCID: "true",
-		"anilistid":                       "105778",
-		"workcid":                         literatureAnchorCardCID,
-		"chapterNumber":                   "3",
-	})
-	// A site card that is already clean: nothing planned.
-	seedRecord(t, c, "clean-site-card", map[string]string{
-		"domain":                        "literature",
-		"fileType":                      "card",
-		"cids/" + literatureSiteCardCID: "true",
-	})
-	// Both locator forms: which one the id belongs to is unknowable, so it
-	// stays.
-	seedRecord(t, c, "merged-card", map[string]string{
-		"domain":                          "literature",
-		"fileType":                        "card",
-		"cids/" + literatureSiteCardCID:   "true",
-		"cids/" + literatureAnchorCardCID: "true",
-		"anilistid":                       "105778",
-	})
-	// No card locator at all: not provably a site card.
-	seedRecord(t, c, "unlocated-card", map[string]string{
-		"domain":                       "literature",
-		"fileType":                     "card",
-		"cids/" + literatureContentCID: "true",
-		"anilistid":                    "105778",
-	})
-	// A stored empty value is still an echo.
-	seedRecord(t, c, "blank-echo-card", map[string]string{
-		"domain":       "literature",
-		"fileType":     "card",
-		"volumeNumber": "",
-	})
-	// A chapter is never swept, whatever it carries.
-	seedRecord(t, c, "chapter", map[string]string{
-		"domain":                       "literature",
-		"fileType":                     "archive",
-		"cids/" + literatureContentCID: "true",
-		"anilistid":                    "105778",
-		"workcid":                      literatureSiteCardCID,
-		"chapterNumber":                "12",
-	})
-	// Not literature: out of scope even though it is a card.
-	seedRecord(t, c, "screen-card", map[string]string{
-		"domain":        "screen",
-		"fileType":      "card",
-		"workcid":       literatureSiteCardCID,
-		"chapterNumber": "1",
-	})
-}
-
-func TestPlanLiteratureEchoes_ScopesToLiteratureCards(t *testing.T) {
-	c, _ := newTestClient(t, "")
-	seedLiteratureCorpus(t, c)
-
-	plans, err := c.planLiteratureEchoes(context.Background())
-	if err != nil {
-		t.Fatalf("plan: %v", err)
-	}
-	got := map[string][]string{}
-	for _, p := range plans {
-		got[p.hashID] = p.fields
-	}
-	want := map[string][]string{
-		"site-card":       {"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"},
-		"anchor-card":     {"workcid", "chapterNumber"},
-		"blank-echo-card": {"volumeNumber"},
-	}
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("plan =\n  %v\nwant\n  %v", got, want)
-	}
-}
-
-func TestSweepLiteratureEchoes_DryRunThenApply(t *testing.T) {
-	c, s := newTestClient(t, "mm:")
-	seedLiteratureCorpus(t, c)
-
-	wantByKey := map[string]int{
-		"anilistid":     1,
-		"workcid":       2,
-		"chapterNumber": 2,
-		"volumeNumber":  2,
-		"chapterStart":  1,
-		"chapterEnd":    1,
-	}
-
-	// Dry run: counts, and nothing written.
-	dry, err := c.SweepLiteratureEchoes(context.Background(), false)
-	if err != nil {
-		t.Fatalf("dry run: %v", err)
-	}
-	if dry.Planned != 3 || dry.Fixed != 0 {
-		t.Errorf("dry run planned/fixed = %d/%d, want 3/0", dry.Planned, dry.Fixed)
-	}
-	if !reflect.DeepEqual(dry.ByKey, wantByKey) {
-		t.Errorf("dry run byKey = %v, want %v", dry.ByKey, wantByKey)
-	}
-	if m, _ := c.GetMetadataFlat("site-card"); m["anilistid"] != "105778" {
-		t.Fatalf("dry run deleted anilistid: %v", m)
-	}
-
-	applied, err := c.SweepLiteratureEchoes(context.Background(), true)
-	if err != nil {
-		t.Fatalf("apply: %v", err)
-	}
-	if applied.Planned != 3 || applied.Fixed != 3 {
-		t.Errorf("apply planned/fixed = %d/%d, want 3/3", applied.Planned, applied.Fixed)
-	}
-	if !reflect.DeepEqual(applied.ByKey, wantByKey) {
-		t.Errorf("apply byKey = %v, want %v", applied.ByKey, wantByKey)
-	}
-
-	site, err := c.GetMetadataFlat("site-card")
-	if err != nil {
-		t.Fatalf("read site-card: %v", err)
-	}
-	for _, k := range []string{"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"} {
-		if _, ok := site[k]; ok {
-			t.Errorf("site-card still carries %s", k)
-		}
-		// The flat key and its field-name index entry must both be gone, or the
-		// next read would MGET a ghost field.
-		if s.Exists("mm:file:site-card/" + k) {
-			t.Errorf("site-card flat key %s survived", k)
-		}
-		if member, _ := s.IsMember("mm:file:site-card/"+FieldsField, k); member {
-			t.Errorf("site-card field index still lists %s", k)
-		}
-	}
-	if site["title"] != "Chainsaw Man" || site["cids/"+literatureSiteCardCID] != "true" {
-		t.Errorf("site-card lost fields it should keep: %v", site)
-	}
-
-	anchor, _ := c.GetMetadataFlat("anchor-card")
-	if anchor["anilistid"] != "105778" {
-		t.Errorf("anchor-card anilistid = %q, want kept", anchor["anilistid"])
-	}
-	if _, ok := anchor["workcid"]; ok {
-		t.Errorf("anchor-card still carries workcid")
-	}
-
-	chapter, _ := c.GetMetadataFlat("chapter")
-	if chapter["anilistid"] != "105778" || chapter["workcid"] != literatureSiteCardCID || chapter["chapterNumber"] != "12" {
-		t.Errorf("chapter was touched: %v", chapter)
-	}
-	screen, _ := c.GetMetadataFlat("screen-card")
-	if screen["workcid"] != literatureSiteCardCID {
-		t.Errorf("screen-card was touched: %v", screen)
-	}
-	merged, _ := c.GetMetadataFlat("merged-card")
-	if merged["anilistid"] != "105778" {
-		t.Errorf("merged-card anilistid = %q, want kept", merged["anilistid"])
-	}
-
-	// Idempotent: the endpoint is called by hand, and a second call must find
-	// nothing — with byKey still an empty map, not nil.
-	again, err := c.SweepLiteratureEchoes(context.Background(), true)
-	if err != nil {
-		t.Fatalf("second pass: %v", err)
-	}
-	if again.Planned != 0 || again.Fixed != 0 || again.ByKey == nil || len(again.ByKey) != 0 {
-		t.Errorf("second pass = %+v, want nothing planned and an empty byKey", again)
-	}
-}

internal/storage/schema_version.go

@@ -1,83 +1,70 @@
 package storage
 
 import (
-	"context"
 	"fmt"
 	"strconv"
-	"time"
 
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
-// SchemaVersion is the Redis-layout version this build of meta-core expects.
+// SchemaVersion is the key-layout version this build of meta-core expects.
 // Distinct from the snapshot manifest version (which is on-disk format);
-// this number describes the in-memory Redis schema — primarily, "are roots
-// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
+// this number describes the record layout — primarily, "are roots UUIDs or
+// content-hash tokens, and do reverse-index keys cid:* exist?"
 //
-// Bump only when a Redis-layout change can't be detected from the data
-// itself or healed by the auto-alias hooks. Each bump is a flag day in
-// alpha: operators wipe and restart.
+// The Redis → SQLite move did not change the record layout (file:<rid>/<field>,
+// cid:<cid>), so it did not bump this: the one-time migration copied records
+// verbatim.
+//
+// Bump only when a layout change can't be detected from the data itself or
+// healed by the write path. Each bump is a flag day in alpha: operators wipe
+// and restart.
 const SchemaVersion = 2
 
-// SchemaVersionKey is the sentinel under which we record SchemaVersion in
-// Redis. Checked on every meta-core boot.
+// SchemaVersionKey is the sentinel under which we record SchemaVersion.
+// Checked on every meta-core boot.
 const SchemaVersionKey = "meta-core:schema-version"
 
-// EnsureSchemaVersion verifies the Redis store matches the expected schema
-// version. Called once at startup, before any other reads/writes.
+// EnsureSchemaVersion verifies the store matches the expected schema version.
+// Called once at startup, before any other reads/writes.
 //
 // Behavior:
 //   - sentinel present and matches expected → OK
-//   - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
-//   - sentinel missing AND file:__index__ non-empty → legacy data, refuse
+//   - sentinel missing AND no roots → fresh store, stamp sentinel
+//   - sentinel missing AND roots exist → legacy data, refuse
 //   - sentinel present but mismatched → refuse, point operator at clean:all
 //
 // Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
 // The error message names the wipe remediation because in alpha there
 // is no automated migration — operators bring up a clean store.
 func (c *Client) EnsureSchemaVersion(expected int) error {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.client == nil {
-		return fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	key := c.buildKey(SchemaVersionKey)
-	have, err := c.client.Get(ctx, key).Result()
-	if err != nil && err != redis.Nil {
+	return c.update(func(tx *kv.Tx) error {
+		ctx := tx.Context()
+		have, ok, err := tx.Get(ctx, SchemaVersionKey)
+		if err != nil {
 			return fmt.Errorf("get schema version: %w", err)
 		}
-
-	if err == redis.Nil {
-		// Sentinel absent. Fresh store iff there's no file:__index__ membership.
-		n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
-		if sErr != nil && sErr != redis.Nil {
-			return fmt.Errorf("scard file:__index__: %w", sErr)
+		if !ok {
+			n, err := tx.Count(ctx, rootsNS)
+			if err != nil {
+				return fmt.Errorf("count roots: %w", err)
 			}
 			if n > 0 {
 				return fmt.Errorf(
-				"redis has %d indexed roots but no %q sentinel — this is alpha "+
-					"data from before the UUID-rooted schema landed. Run "+
-					"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
-					"to wipe and restart, or downgrade meta-core to a compatible build",
+					"the store has %d indexed roots but no %q sentinel — this is alpha "+
+						"data from before the UUID-rooted schema landed. Delete the "+
+						"database file to wipe and restart, or downgrade meta-core to a compatible build",
 					n, SchemaVersionKey)
 			}
-		if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
-			return fmt.Errorf("set schema version: %w", err)
+			return tx.Put(SchemaVersionKey, strconv.Itoa(expected))
 		}
-		return nil
-	}
-
 		if have == strconv.Itoa(expected) {
 			return nil
 		}
 		return fmt.Errorf(
-		"redis schema version is %q, this build of meta-core expects %d. "+
-			"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
+			"store schema version is %q, this build of meta-core expects %d. "+
+				"Delete the database file to wipe and restart, or use a meta-core "+
 				"build that matches the existing data",
 			have, expected)
+	})
 }

internal/storage/search.go

@@ -0,0 +1,565 @@
+package storage
+
+// Search over the KV store.
+//
+// The Redis era answered POST /api/metadata/search from an in-memory copy of
+// every record (the API's searchIndex), because Redis could not filter
+// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
+// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
+// gone: search now runs against two derived structures inside the database.
+//
+//   - kv_field_v — an expression index over (lowercased field name parsed out
+//     of the key, lowercased value, value), covering only record keys. It answers "which
+//     records have field F with a value like V" for ANY field, including ones
+//     invented tomorrow: the data stays schema-free, the index is over the key
+//     itself.
+//   - fts — an FTS5 trigram table holding one row per record: the record's
+//     haystack (buildHaystack) plus its rid. Trigrams make every substring of
+//     3+ characters indexable, which is the matching rule search always had.
+//
+// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
+// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
+// semantics are bit-identical to the in-memory matcher and nothing depends on
+// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+// The field expression and the partial-index predicate MUST appear verbatim
+// (as literals, never bound parameters) in every query meant to use kv_field_v —
+// SQLite only matches a partial index whose WHERE clause it can see implied by
+// the query text. TestSearchFilterUsesFieldIndex pins this.
+const (
+	fieldExpr  = `lower(substr(k, instr(k,'/')+1))`
+	recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
+	ridExpr    = `substr(k, 6, instr(k,'/')-6)`
+)
+
+// searchSchema is the derived DDL applied when the database is opened.
+//
+// kv_field_v carries the raw value as a third column so the filters' substring
+// test (instr(lower(v), …)) is answered from the index alone. Without it,
+// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
+// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
+// lower(v)), which is dropped here; the rebuild costs ~80 s once.
+const searchSchema = `
+DROP INDEX IF EXISTS kv_field;
+CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
+CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
+`
+
+// searchIndexFields are the metadata fields folded into a record's search
+// haystack — must stay in sync with the fields the search handler documents.
+var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
+
+// affectsHaystack reports whether writing field can change the haystack.
+func affectsHaystack(field string) bool {
+	if strings.HasPrefix(field, "titles/") {
+		return true
+	}
+	for _, f := range searchIndexFields {
+		if field == f {
+			return true
+		}
+	}
+	return false
+}
+
+// Haystack folds a record's searchable text fields into one lowercased blob —
+// what free-text tokens must each be a substring of.
+//
+// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
+// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
+// sentinel "true", so a search for an alternative title (`3 percent`) finds the
+// record whose `title` is `3%`.
+func Haystack(fields map[string]string) string {
+	var sb strings.Builder
+	for _, f := range searchIndexFields {
+		if v, ok := fields[f]; ok && v != "" {
+			sb.WriteString(strings.ToLower(v))
+			sb.WriteByte('\n')
+		}
+	}
+	for _, name := range titleMemberNames(fields) {
+		sb.WriteString(strings.ToLower(name))
+		sb.WriteByte('\n')
+	}
+	return sb.String()
+}
+
+// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
+// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
+// back. Sorted so the haystack is deterministic.
+func titleMemberNames(fields map[string]string) []string {
+	var names []string
+	for k, v := range fields {
+		rest, ok := strings.CutPrefix(k, "titles/")
+		if !ok || v == "" || v == "false" {
+			continue
+		}
+		_, name, ok := strings.Cut(rest, "/")
+		if !ok || name == "" {
+			continue
+		}
+		names = append(names, strings.ReplaceAll(name, "∕", "/"))
+	}
+	sort.Strings(names)
+	return names
+}
+
+// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
+// "query is a substring of the hashID" rule also has an indexed candidate path.
+// Verification checks the two separately (a rid token plus a title token must
+// not match together).
+func ftsHay(hashID string, fields map[string]string) string {
+	return Haystack(fields) + strings.ToLower(hashID) + "\n"
+}
+
+// refreshSearchTx rewrites a record's FTS row from its current fields, or
+// drops it when the record is gone.
+func refreshSearchTx(tx *kv.Tx, hashID string) error {
+	fields, err := readRecord(tx.Context(), tx, hashID)
+	if err != nil {
+		return err
+	}
+	if len(fields) == 0 {
+		return deleteSearchRowTx(tx, hashID)
+	}
+	return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
+}
+
+// ensureSearchRowTx creates the record's FTS row if it has none (a record
+// whose writes never touched a haystack field still needs one for the rid
+// rule).
+func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
+	_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
+	if err != nil || ok {
+		return err
+	}
+	return refreshSearchTx(tx, hashID)
+}
+
+func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
+	rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
+	if err != nil {
+		return err
+	}
+	if ok {
+		if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
+			return fmt.Errorf("fts update %s: %w", hashID, err)
+		}
+		return nil
+	}
+	res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
+	if err != nil {
+		return fmt.Errorf("fts insert %s: %w", hashID, err)
+	}
+	rowid, err := res.LastInsertId()
+	if err != nil {
+		return err
+	}
+	return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
+}
+
+func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
+	rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
+	if err != nil || !ok {
+		return err
+	}
+	if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
+		return fmt.Errorf("fts delete %s: %w", hashID, err)
+	}
+	_, err = tx.Delete(ftsNS + hashID)
+	return err
+}
+
+// SearchFilter is one structured filter: AND across filters, OR across Values.
+type SearchFilter struct {
+	Property string
+	Values   []string
+}
+
+// SearchHit is one verified result.
+type SearchHit struct {
+	HashID string
+	Fields map[string]string
+}
+
+// Search returns up to limit records, in hashID order, that satisfy match.
+//
+// query and filters only NARROW the candidates (SQL, superset semantics);
+// match is the authoritative test, run on each hydrated record. Pass the same
+// query/filters match evaluates, or results will be missing.
+func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
+	s, err := c.kv()
+	if err != nil {
+		return nil, err
+	}
+	if limit <= 0 {
+		limit = 100
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+
+	driver, probes, err := c.planCandidates(ctx, s, query, filters)
+	if err != nil {
+		return nil, err
+	}
+	page := max(limit*2, 500)
+	hits := make([]SearchHit, 0, limit)
+	after := ""
+	for len(hits) < limit {
+		ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
+		if err != nil {
+			return nil, err
+		}
+		survivors := ids
+		if len(probes) > 0 && len(ids) > 0 {
+			if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
+				return nil, err
+			}
+		}
+		for _, id := range survivors {
+			fields, err := readRecord(ctx, s, id)
+			if err != nil {
+				return nil, err
+			}
+			if fields == nil || !match(id, fields) {
+				continue
+			}
+			hits = append(hits, SearchHit{HashID: id, Fields: fields})
+			if len(hits) >= limit {
+				break
+			}
+		}
+		if len(ids) < page {
+			break
+		}
+		after = ids[len(ids)-1]
+	}
+	return hits, nil
+}
+
+// queryIDs returns one page of the driver's candidates, in rid order.
+func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
+	q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
+	rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
+	if err != nil {
+		return nil, fmt.Errorf("search candidates: %w", err)
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		out = append(out, id)
+	}
+	return out, rows.Err()
+}
+
+// probeSQL applies the non-driving filters to one page of candidate rids
+// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
+func probeSQL(probes []candidatePart) (string, []any) {
+	where := make([]string, len(probes))
+	var args []any
+	for i, p := range probes {
+		where[i] = p.exists
+		args = append(args, p.existsArgs...)
+	}
+	return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
+}
+
+// probeIDs keeps the ids (in their order) that pass every probe.
+func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
+	q, args := probeSQL(probes)
+	list, err := json.Marshal(ids)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
+	if err != nil {
+		return nil, fmt.Errorf("search probes: %w", err)
+	}
+	defer rows.Close()
+	pass := make(map[string]bool, len(ids))
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		pass[id] = true
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	out := ids[:0:0]
+	for _, id := range ids {
+		if pass[id] {
+			out = append(out, id)
+		}
+	}
+	return out, nil
+}
+
+// candidatePart is one narrowing condition: free text, or one filter.
+type candidatePart struct {
+	sql  string // SELECT … AS rid
+	args []any
+	// exists is the same filter as a predicate on candidate row c.rid (empty
+	// for free text, which has no per-record probe: the FTS rid column is not
+	// indexed).
+	exists     string
+	existsArgs []any
+}
+
+// estimateCap bounds the count used to pick the driving part: it only has to
+// tell a 50-row part from a 100k-row one.
+const estimateCap = 1000
+
+// planCandidates picks how to narrow a search: one DRIVING part (the most
+// selective, by a capped count) that lists candidate rids in order, plus PROBES
+// — every other filter, checked as an EXISTS over each candidate's own key
+// range (a primary-key range read of ~40 rows).
+//
+// Search pages the driver in Go and probes one page at a time, so a broad
+// query stops as soon as `limit` records verify. Two earlier shapes were
+// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
+// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
+// and probing inside the driver query (SQLite evaluates the probe on every
+// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
+// both ~100k, 10 s). Free text that does not drive is dropped from SQL
+// entirely — the verifier still applies it, and dropping a condition only
+// widens the superset.
+func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
+	var parts []candidatePart
+	if p, ok := textCandidates(query); ok {
+		parts = append(parts, p)
+	}
+	for _, f := range filters {
+		if p, ok := filterCandidates(f); ok {
+			parts = append(parts, p)
+		}
+	}
+	if len(parts) == 0 {
+		return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
+	}
+
+	driver := 0
+	if len(parts) > 1 {
+		best := int64(-1)
+		for i, p := range parts {
+			var n int64
+			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
+			if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
+				return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
+			}
+			if best < 0 || n < best {
+				best, driver = n, i
+			}
+		}
+	}
+	var probes []candidatePart
+	for i, p := range parts {
+		if i != driver && p.exists != "" {
+			probes = append(probes, p)
+		}
+	}
+	return parts[driver], probes, nil
+}
+
+// candidateSQL is planCandidates without the store: text plus filters, the
+// first part driving. Used by tests that inspect the plan.
+func candidateSQL(query string, filters []SearchFilter) (string, []any) {
+	var parts []candidatePart
+	if p, ok := textCandidates(query); ok {
+		parts = append(parts, p)
+	}
+	for _, f := range filters {
+		if p, ok := filterCandidates(f); ok {
+			parts = append(parts, p)
+		}
+	}
+	if len(parts) == 0 {
+		return "", nil
+	}
+	return parts[0].sql, parts[0].args
+}
+
+// textCandidates narrows by free text: every whitespace token must be a
+// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
+// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
+// same rows.
+func textCandidates(query string) (candidatePart, bool) {
+	tokens := strings.Fields(strings.ToLower(query))
+	if len(tokens) == 0 {
+		return candidatePart{}, false
+	}
+	var match []string
+	var likes []string
+	var likeArgs []any
+	for _, t := range tokens {
+		if utf8.RuneCountInString(t) >= 3 {
+			match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
+		} else {
+			likes = append(likes, `hay LIKE ? ESCAPE '\'`)
+			likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
+		}
+	}
+	var where []string
+	var args []any
+	if len(match) > 0 {
+		where = append(where, `fts MATCH ?`)
+		args = append(args, strings.Join(match, " "))
+	}
+	where = append(where, likes...)
+	args = append(args, likeArgs...)
+	return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
+}
+
+func escapeLike(s string) string {
+	r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
+	return r.Replace(s)
+}
+
+// filterCandidates narrows by one filter — the UNION, over its values, of the
+// two storage shapes recordMatchesFilter probes:
+//
+//   - key-set member `<prop>/<value>` (any case): the lowercased field equals
+//     lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
+//     lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
+//     comparison could miss);
+//   - flat field `<prop>` whose lowercased value contains the lowercased value
+//     — covers exact and contains alike, and comma-split csv-set values.
+//
+// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
+// name): it constrains nothing in SQL and is left to the verifier.
+func filterCandidates(f SearchFilter) (candidatePart, bool) {
+	var values []string
+	for _, v := range f.Values {
+		if v = strings.TrimSpace(v); v != "" {
+			values = append(values, v)
+		}
+	}
+	if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
+		return candidatePart{}, false
+	}
+	prop := asciiLower(f.Property)
+	var conds []string
+	var args []any
+	memberRange := false
+	for _, v := range values {
+		if isASCII(v) {
+			lv := asciiLower(v)
+			conds = append(conds, fieldExpr+` = ?`)
+			args = append(args, prop+"/"+lv)
+			conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
+			args = append(args, prop, lv)
+		} else {
+			memberRange = true
+			conds = append(conds, fieldExpr+` = ?`)
+			args = append(args, prop)
+		}
+	}
+	if memberRange {
+		conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
+		args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
+	}
+
+	// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
+	// OR inside one SELECT would defeat the partial index; plain UNION would
+	// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
+	// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
+	base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
+	selects := make([]string, len(conds))
+	for i, cond := range conds {
+		selects[i] = base + cond
+	}
+	// As a probe: the same conditions over one candidate's key range.
+	exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
+		strings.Join(conds, ") OR (") + `)))`
+	return candidatePart{
+		sql: strings.Join(selects, " UNION ALL "), args: args,
+		exists: exists, existsArgs: args,
+	}, true
+}
+
+func isASCII(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= utf8.RuneSelf {
+			return false
+		}
+	}
+	return true
+}
+
+// asciiLower lowercases A–Z only, exactly like SQLite's lower().
+func asciiLower(s string) string {
+	b := []byte(s)
+	for i, c := range b {
+		if 'A' <= c && c <= 'Z' {
+			b[i] = c + ('a' - 'A')
+		}
+	}
+	return string(b)
+}
+
+func envInt(name string, def int) int {
+	if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
+		return v
+	}
+	return def
+}
+
+// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
+// empty but records exist — a store restored from a snapshot, a copy made
+// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
+// change its tokenizer. The rows are derived, so this is a repair, not a
+// migration. Runs in bounded batches so writes keep flowing; until it
+// finishes, free-text search under-matches (filters are unaffected).
+func (c *Client) RebuildSearchIfEmpty() (int, error) {
+	s, err := c.kv()
+	if err != nil {
+		return 0, err
+	}
+	ctx, cancel := opCtx()
+	var rows int64
+	err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
+	cancel()
+	if err != nil {
+		return 0, fmt.Errorf("count fts rows: %w", err)
+	}
+	if rows > 0 {
+		return 0, nil
+	}
+	ids, err := c.GetAllHashIDs()
+	if err != nil {
+		return 0, err
+	}
+	const batch = 500
+	for start := 0; start < len(ids); start += batch {
+		chunk := ids[start:min(start+batch, len(ids))]
+		if err := c.update(func(tx *kv.Tx) error {
+			for _, id := range chunk {
+				if err := refreshSearchTx(tx, id); err != nil {
+					return err
+				}
+			}
+			return nil
+		}); err != nil {
+			return start, fmt.Errorf("rebuild search rows: %w", err)
+		}
+	}
+	return len(ids), nil
+}

internal/storage/search_dirty.go

@@ -1,245 +0,0 @@
-package storage
-
-import (
-	"context"
-	"fmt"
-	"log"
-	"sort"
-	"strings"
-	"time"
-)
-
-// Incremental search-index maintenance.
-//
-// # Why this exists
-//
-// The API's in-memory search index used to be rebuilt in full every 60 s:
-// GetSearchIndexTuples does a whole-keyspace SCAN plus batched MGETs, builds an
-// intermediate map of every record, then allocates a fresh []SearchTuple — all
-// while the previous index is still live serving reads. That cost is O(corpus),
-// not O(change): adding 200 records in a minute re-read every one of the other
-// ~620 000 field keys to notice them.
-//
-// Measured on a 19 622-record staging corpus (2026-08-30): SCAN was 57.8 % and
-// MGET 19.1 % of ALL Redis CPU, and ~90 % of the MGET calls plus ~48 % of the
-// SCAN calls were this loop — roughly 45 % of Redis's total work, spent
-// continuously whether or not anything changed. The Go side held 783 MB for
-// data Redis stores in 135 MB, largely because of the double copy.
-//
-// That was tolerable while the corpus was small and mostly static. It stops
-// being tolerable now that meta-core is the platform's permanent, additive
-// store: every search hit on every gateway is retained forever, so the rebuild
-// cost is a ratchet.
-//
-// # The shape
-//
-// Every write path marks its record in a dirty set; the refresh drains that set
-// and re-reads only those records. Cost becomes O(change).
-//
-// Marking rides along with the field-index maintenance every write path already
-// performs, so there is no new bookkeeping discipline to remember — if you are
-// adding a write path, the same line that keeps `__fields__` correct keeps this
-// correct.
-//
-// # Correctness
-//
-// The drain is a destructive SPOP, so a write that lands *during* a refresh is
-// either already in the drained batch or still in the set for the next tick —
-// never dropped. The cost of the race is one tick of staleness, which is well
-// inside the interval the index was already stale by.
-//
-// A refresh that fails after draining WOULD lose its batch, so drainDirty's
-// caller re-marks on error (see MarkDirty / the API's refreshIncremental).
-//
-// ⚠ **The destructive drain assumes ONE index consumer per Redis.** Two
-// meta-core processes sharing a keyspace would steal each other's marks, and
-// each would end up with an index missing whatever the other drained. That is
-// fine today — multi-instance leader election was deprecated and the dev/prod
-// stacks run a single meta-core — but it is the assumption to revisit first if
-// a second instance is ever pointed at the same Redis. A per-consumer dirty key
-// (`file:__dirty__:<consumer>`) is the shape that would fix it.
-//
-// The API's periodic full reconcile (searchIndexReconcileEvery) is the backstop
-// for both this and for a write path that forgets to mark: a missed mark costs
-// staleness until the next reconcile, never permanent divergence.
-
-// dirtyField is the reserved key holding the set of record ids whose metadata
-// changed since the last search-index refresh. It sits under the same `file:`
-// namespace as `__index__` and is likewise not a record.
-const dirtyField = "file:__dirty__"
-
-// dirtyDrainCap bounds one incremental refresh. A burst larger than this is
-// drained over several ticks rather than in one long-held pass — the point of
-// the change is to keep any single refresh cheap. Sized well above a normal
-// search-persist flush (which writes at most a few hundred records).
-const dirtyDrainCap = 5000
-
-// buildDirtyKey constructs the key for the dirty-record set.
-func (c *Client) buildDirtyKey() string {
-	return c.buildKey(dirtyField)
-}
-
-// markDirtyLocked records that hashID's metadata changed. Caller must hold c.mu.
-//
-// Best-effort by design: a failure here costs one record its incremental
-// refresh, and the periodic full rebuild still repairs it. Failing the caller's
-// write instead would trade a stale index entry for a lost write, which is
-// strictly worse.
-func (c *Client) markDirtyLocked(ctx context.Context, hashID string) {
-	if hashID == "" {
-		return
-	}
-	if err := c.client.SAdd(ctx, c.buildDirtyKey(), hashID).Err(); err != nil {
-		log.Printf("[STORAGE] dirty-mark failed for %s: %v (index repairs on the next full rebuild)", hashID, err)
-	}
-}
-
-// MarkDirty records that hashID needs re-indexing. Exported for the API layer,
-// which re-marks a batch whose refresh failed after the drain.
-func (c *Client) MarkDirty(hashIDs ...string) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.client == nil || len(hashIDs) == 0 {
-		return
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	members := make([]interface{}, 0, len(hashIDs))
-	for _, h := range hashIDs {
-		if h != "" {
-			members = append(members, h)
-		}
-	}
-	if len(members) == 0 {
-		return
-	}
-	if err := c.client.SAdd(ctx, c.buildDirtyKey(), members...).Err(); err != nil {
-		log.Printf("[STORAGE] dirty re-mark failed for %d records: %v", len(members), err)
-	}
-}
-
-// DrainDirty removes and returns up to dirtyDrainCap record ids from the dirty
-// set. `more` reports whether entries remain, so the caller can schedule
-// another pass instead of waiting a full interval to catch up on a burst.
-//
-// Destructive on purpose: SPOP makes concurrent refreshes disjoint and stops a
-// record being re-read every tick forever. The caller owns the drained ids
-// until it has indexed them — on failure it must MarkDirty them again.
-func (c *Client) DrainDirty() (ids []string, more bool, err error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, false, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-
-	key := c.buildDirtyKey()
-	ids, err = c.client.SPopN(ctx, key, dirtyDrainCap).Result()
-	if err != nil {
-		return nil, false, fmt.Errorf("spop dirty: %w", err)
-	}
-	if len(ids) < dirtyDrainCap {
-		return ids, false, nil
-	}
-	remaining, err := c.client.SCard(ctx, key).Result()
-	if err != nil {
-		// We have the batch; not knowing the remainder just means the next
-		// tick picks it up on schedule.
-		return ids, false, nil
-	}
-	return ids, remaining > 0, nil
-}
-
-// GetSearchIndexTuplesFor reads the search tuples for a specific set of records
-// — the incremental counterpart to GetSearchIndexTuples.
-//
-// Reads go through the per-record field index (GetMetadataFlat), so this is two
-// round-trips per record rather than a keyspace scan. That trade is what makes
-// it O(change): it beats the full pass whenever the changed set is a small
-// fraction of the corpus, and the caller bounds the batch (dirtyDrainCap).
-//
-// A record present in the dirty set but absent from storage was deleted; it is
-// reported in `gone` so the caller can evict it from the index rather than
-// leaving a tombstone that still answers searches.
-func (c *Client) GetSearchIndexTuplesFor(hashIDs []string) (tuples []SearchTuple, gone []string, err error) {
-	if len(hashIDs) == 0 {
-		return nil, nil, nil
-	}
-
-	excluded := indexExcludePrefixes()
-	tuples = make([]SearchTuple, 0, len(hashIDs))
-
-	for _, hashID := range hashIDs {
-		fields, ferr := c.GetMetadataFlat(hashID)
-		if ferr != nil {
-			// Re-mark so the record is retried rather than silently missing
-			// from the index until the next full rebuild.
-			c.MarkDirty(hashID)
-			log.Printf("[STORAGE] incremental index read failed for %s: %v", hashID, ferr)
-			continue
-		}
-		if len(fields) == 0 {
-			gone = append(gone, hashID)
-			continue
-		}
-		for field := range fields {
-			if hasAnyPrefix(field, excluded) {
-				delete(fields, field)
-			}
-		}
-		tuples = append(tuples, SearchTuple{
-			HashID:   hashID,
-			Haystack: buildHaystack(fields),
-			Fields:   fields,
-		})
-	}
-	return tuples, gone, nil
-}
-
-// buildHaystack folds a record's searchable text fields into one lowercased
-// blob. Shared by the full and incremental index reads so the two can never
-// disagree about what is searchable — a drift there would make a record's
-// matchability depend on which path happened to index it.
-//
-// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
-// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
-// sentinel "true", so a search for an alternative title (`3 percent`) finds the
-// record whose `title` is `3%`.
-func buildHaystack(fields map[string]string) string {
-	var sb strings.Builder
-	for _, f := range searchIndexFields {
-		if v, ok := fields[f]; ok && v != "" {
-			sb.WriteString(strings.ToLower(v))
-			sb.WriteByte('\n')
-		}
-	}
-	for _, name := range titleMemberNames(fields) {
-		sb.WriteString(strings.ToLower(name))
-		sb.WriteByte('\n')
-	}
-	return sb.String()
-}
-
-// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
-// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
-// back. Sorted: the fields map has no order, and the full and incremental
-// builds must produce byte-identical haystacks.
-func titleMemberNames(fields map[string]string) []string {
-	var names []string
-	for k, v := range fields {
-		rest, ok := strings.CutPrefix(k, "titles/")
-		if !ok || v == "" || v == "false" {
-			continue
-		}
-		_, name, ok := strings.Cut(rest, "/")
-		if !ok || name == "" {
-			continue
-		}
-		names = append(names, strings.ReplaceAll(name, "\u2215", "/"))
-	}
-	sort.Strings(names)
-	return names
-}

internal/storage/search_dirty_test.go

@@ -1,230 +0,0 @@
-package storage
-
-import (
-	"context"
-	"sort"
-	"testing"
-)
-
-// Tests for incremental search-index maintenance (search_dirty.go).
-//
-// The dirty set is derived state on the hot write path, which makes its failure
-// mode the dangerous kind: a MISSING mark doesn't error, it just leaves the
-// search index quietly serving a stale copy of that record until someone
-// triggers a full rebuild. So the property worth pinning is coverage — every
-// path that changes what a search would match must mark — and that is what the
-// bulk of these tests assert, one write path at a time.
-//
-// The second property is that a delete is a mark too, not an unmark: the
-// refresh has to LEARN the record is gone in order to evict it. Marking on
-// delete is counter-intuitive enough to be worth a test of its own.
-
-func dirtyMembers(t *testing.T, c *Client) []string {
-	t.Helper()
-	got, err := c.client.SMembers(context.Background(), c.buildDirtyKey()).Result()
-	if err != nil {
-		t.Fatalf("smembers dirty: %v", err)
-	}
-	sort.Strings(got)
-	return got
-}
-
-func clearDirty(t *testing.T, c *Client) {
-	t.Helper()
-	if err := c.client.Del(context.Background(), c.buildDirtyKey()).Err(); err != nil {
-		t.Fatalf("del dirty: %v", err)
-	}
-}
-
-// Every write path must mark. A path that doesn't leaves the index serving a
-// stale record with no error anywhere.
-func TestDirty_EveryWritePathMarks(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		cases := []struct {
-			name  string
-			id    string
-			write func(id string) error
-		}{
-			{"SetMetadataFlat", "r-set", func(id string) error {
-				return c.SetMetadataFlat(id, map[string]string{"title": "A"})
-			}},
-			{"SetProperty", "r-prop", func(id string) error {
-				return c.SetProperty(id, "title", "B")
-			}},
-			{"MergeMetadataFlat", "r-merge", func(id string) error {
-				_, err := c.MergeMetadataFlat(id, map[string]string{"title": "C"})
-				return err
-			}},
-			{"AddToSet", "r-add", func(id string) error {
-				_, err := c.AddToSet(id, "tags", "x")
-				return err
-			}},
-		}
-		for _, tc := range cases {
-			t.Run(tc.name, func(t *testing.T) {
-				clearDirty(t, c)
-				if err := tc.write(tc.id); err != nil {
-					t.Fatalf("%s: %v", tc.name, err)
-				}
-				got := dirtyMembers(t, c)
-				if len(got) != 1 || got[0] != tc.id {
-					t.Fatalf("%s must mark %q dirty, got %v", tc.name, tc.id, got)
-				}
-			})
-		}
-	})
-}
-
-// RemoveFromSet's non-empty branch changes a field's VALUE without changing the
-// field-name index, so it is the one path that a mark riding on field-index
-// maintenance would have missed. Pinned explicitly for that reason.
-func TestDirty_RemoveFromSetMarksEvenWhenTheFieldSurvives(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		const id = "r-rem"
-		for _, v := range []string{"x", "y"} {
-			if _, err := c.AddToSet(id, "tags", v); err != nil {
-				t.Fatalf("AddToSet: %v", err)
-			}
-		}
-		clearDirty(t, c)
-
-		if _, err := c.RemoveFromSet(id, "tags", "x"); err != nil {
-			t.Fatalf("RemoveFromSet: %v", err)
-		}
-		// `tags` still exists (y remains) — no field-index change, but the
-		// searchable value changed.
-		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
-			t.Fatalf("a value-only change must still mark dirty, got %v", got)
-		}
-	})
-}
-
-// A delete MARKS rather than unmarks: the refresh must read the record, find it
-// gone, and evict it. Unmarking would leave a deleted record answering searches
-// until the next full rebuild.
-func TestDirty_DeletesMarkSoTheRefreshCanEvict(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		const id = "r-del"
-		if err := c.SetMetadataFlat(id, map[string]string{"title": "Doomed"}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-		clearDirty(t, c)
-
-		if _, err := c.DeleteMetadata(id); err != nil {
-			t.Fatalf("DeleteMetadata: %v", err)
-		}
-		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
-			t.Fatalf("a delete must mark dirty so the index evicts it, got %v", got)
-		}
-
-		// …and the incremental read reports it as gone rather than as an error
-		// or an empty-but-present tuple.
-		tuples, gone, err := c.GetSearchIndexTuplesFor([]string{id})
-		if err != nil {
-			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
-		}
-		if len(tuples) != 0 {
-			t.Fatalf("a deleted record must produce no tuple, got %d", len(tuples))
-		}
-		if len(gone) != 1 || gone[0] != id {
-			t.Fatalf("a deleted record must be reported gone, got %v", gone)
-		}
-	})
-}
-
-// The drain is destructive so concurrent refreshes stay disjoint and a record
-// isn't re-read every tick forever. The caller owns what it drained.
-func TestDirty_DrainIsDestructive(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		clearDirty(t, c)
-		c.MarkDirty("a", "b", "c")
-
-		ids, more, err := c.DrainDirty()
-		if err != nil {
-			t.Fatalf("DrainDirty: %v", err)
-		}
-		sort.Strings(ids)
-		if len(ids) != 3 {
-			t.Fatalf("expected 3 drained ids, got %v", ids)
-		}
-		if more {
-			t.Fatal("a fully-drained set must not report more")
-		}
-		if got := dirtyMembers(t, c); len(got) != 0 {
-			t.Fatalf("drain must empty the set, got %v", got)
-		}
-
-		// A second drain is empty, not an error — the steady state when nothing
-		// has been written since the last tick.
-		ids, _, err = c.DrainDirty()
-		if err != nil || len(ids) != 0 {
-			t.Fatalf("draining an empty set must be a clean no-op, got %v / %v", ids, err)
-		}
-	})
-}
-
-// The incremental read must produce tuples indistinguishable from the full
-// rebuild's — same haystack, same fields, same exclusions. A drift here would
-// make a record's matchability depend on which path happened to index it, which
-// is invisible until someone searches for the one term that differs.
-func TestDirty_IncrementalTuplesMatchTheFullRebuild(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		const id = "r-parity"
-		if err := c.SetMetadataFlat(id, map[string]string{
-			"title":                   "Parity Release",
-			"fileName":                "parity.s01e01.mkv",
-			"fileType":                "video",
-			"titles/eng/3 Percent":    "true",
-			"titles/fra/3 Pourcent":   "true",
-			"categories/XXX":          "true",
-			"categories/newznab/5070": "true",
-		}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		full := tuplesByHashID(t, c)[id]
-		incr, gone, err := c.GetSearchIndexTuplesFor([]string{id})
-		if err != nil {
-			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
-		}
-		if len(gone) != 0 || len(incr) != 1 {
-			t.Fatalf("expected exactly one live tuple, got %d live / %d gone", len(incr), len(gone))
-		}
-
-		if incr[0].Haystack != full.Haystack {
-			t.Errorf("haystack drift:\n incremental %q\n full        %q", incr[0].Haystack, full.Haystack)
-		}
-		if len(incr[0].Fields) != len(full.Fields) {
-			t.Errorf("field-count drift: incremental %d, full %d", len(incr[0].Fields), len(full.Fields))
-		}
-		for k, v := range full.Fields {
-			if incr[0].Fields[k] != v {
-				t.Errorf("field %q drift: incremental %q, full %q", k, incr[0].Fields[k], v)
-			}
-		}
-		// The index projection must apply on both paths.
-		if _, ok := incr[0].Fields["categories/newznab/5070"]; ok {
-			t.Error("the incremental path must honour the index exclusion list too")
-		}
-		if _, ok := incr[0].Fields["categories/XXX"]; !ok {
-			t.Error("the incremental path must keep the adult-filter category")
-		}
-	})
-}
-
-// A `titles/<lang3>/<name>` member contributes its NAME (the key), never its
-// "true" value, and the `∕` a writer substituted for `/` reads back as `/`.
-func TestBuildHaystack_FoldsTitleMemberNames(t *testing.T) {
-	got := buildHaystack(map[string]string{
-		"title":                     "3%",
-		"titles/fra/3 Pourcent":     "true",
-		"titles/eng/3 Percent":      "true",
-		"titles/und/Fate\u2215Zero": "true",
-		"titles/eng":                "Retired Scalar",
-		"titles/eng/":               "true",
-	})
-	want := "3%\n3 percent\n3 pourcent\nfate/zero\n"
-	if got != want {
-		t.Errorf("haystack = %q, want %q", got, want)
-	}
-}

internal/storage/search_index_exclude_test.go

@@ -1,116 +0,0 @@
-package storage
-
-import (
-	"testing"
-)
-
-// Tests for the search-index field projection (GetSearchIndexTuples +
-// indexExcludePrefixes).
-//
-// The projection narrows the in-memory index only — Redis keeps every field and
-// every per-record read still returns it. What has to be pinned is the boundary
-// between the two, because getting it wrong in either direction is silent:
-// exclude too little and the index keeps paying for fields nothing reads;
-// exclude too much and a record served *from the index* comes back missing a
-// field a downstream consumer depends on.
-//
-// The specific hazard the default guards is adult-content filtering:
-// `categories/XXX` is the server-side enforcement point (meta-search's
-// `query_eval::fields_are_adult`, meta-gateway's `cards::fields_are_adult`),
-// and `exclude_adult` defaults to ON. Excluding the whole `categories/` family
-// would make every index-served record read as non-adult and the filter would
-// fail OPEN — so the default must keep `categories/XXX` while dropping the
-// reader-less `categories/newznab/` mirror.
-
-func tuplesByHashID(t *testing.T, c *Client) map[string]SearchTuple {
-	t.Helper()
-	got, err := c.GetSearchIndexTuples()
-	if err != nil {
-		t.Fatalf("GetSearchIndexTuples: %v", err)
-	}
-	out := make(map[string]SearchTuple, len(got))
-	for _, tu := range got {
-		out[tu.HashID] = tu
-	}
-	return out
-}
-
-func TestSearchIndex_ExcludesNewznabCategoriesButKeepsTheRest(t *testing.T) {
-	forEachPrefix(t, func(t *testing.T, c *Client) {
-		const id = "root-1"
-		if err := c.SetMetadataFlat(id, map[string]string{
-			"title":                   "Some Release",
-			"fileType":                "video",
-			"categories/TV/Anime":     "true",
-			"categories/XXX":          "true",
-			"categories/newznab/5070": "true",
-			"categories/newznab/2040": "true",
-		}); err != nil {
-			t.Fatalf("SetMetadataFlat: %v", err)
-		}
-
-		fields := tuplesByHashID(t, c)[id].Fields
-		if fields == nil {
-			t.Fatal("record missing from the search index")
-		}
-
-		for _, excluded := range []string{"categories/newznab/5070", "categories/newznab/2040"} {
-			if _, ok := fields[excluded]; ok {
-				t.Errorf("%s must not be indexed — it is a reader-less mirror vocabulary", excluded)
-			}
-		}
-		// The load-bearing ones must survive. XXX especially: without it the
-		// adult filter fails open on every index-served record.
-		for _, kept := range []string{"categories/XXX", "categories/TV/Anime", "title", "fileType"} {
-			if _, ok := fields[kept]; !ok {
-				t.Errorf("%s must stay in the index", kept)
-			}
-		}
-
-		// The store itself is untouched — the projection is index-only.
-		stored, err := c.GetMetadataFlat(id)
-		if err != nil {
-			t.Fatalf("GetMetadataFlat: %v", err)
-		}
-		if _, ok := stored["categories/newznab/5070"]; !ok {
-			t.Error("an index-excluded field must still be stored and readable per-record")
-		}
-	})
-}
-
-func TestIndexExcludePrefixes_EnvOverride(t *testing.T) {
-	if got := indexExcludePrefixes(); len(got) != 1 || got[0] != "categories/newznab/" {
-		t.Fatalf("unset env must yield the default, got %v", got)
-	}
-
-	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "foo/, bar/ ,")
-	got := indexExcludePrefixes()
-	if len(got) != 2 || got[0] != "foo/" || got[1] != "bar/" {
-		t.Fatalf("override must trim and drop blanks, got %v", got)
-	}
-
-	// An explicitly empty value is the escape hatch: index everything.
-	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "")
-	if got := indexExcludePrefixes(); len(got) != 0 {
-		t.Fatalf("an empty override must disable exclusion, got %v", got)
-	}
-}
-
-func TestHasAnyPrefix(t *testing.T) {
-	prefixes := []string{"categories/newznab/", "tmp/"}
-	cases := map[string]bool{
-		"categories/newznab/5070": true,
-		"tmp/scratch":             true,
-		"categories/XXX":          false,
-		"categories/newznab":      false, // the family root is not the mirror
-		"title":                   false,
-	}
-	for field, want := range cases {
-		if got := hasAnyPrefix(field, prefixes); got != want {
-			t.Errorf("hasAnyPrefix(%q) = %v, want %v", field, got, want)
-		}
-	}
-	if hasAnyPrefix("anything", nil) {
-		t.Error("an empty prefix list must exclude nothing")
-	}
-}

internal/storage/search_test.go

@@ -0,0 +1,327 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+// These tests pin the one property storage.Search must have: its SQL
+// candidates are a SUPERSET of the records the exact matcher accepts. A
+// candidate miss is a silently dropped search result, so each case below is a
+// shape where SQLite and Go disagree about text (case folding, csv values,
+// FTS syntax, trigram length) and the candidate query has to cover for it.
+//
+// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
+// every candidate path; the API tests own the matcher itself.
+
+func seed(t *testing.T, c *Client, records map[string]map[string]string) {
+	t.Helper()
+	for id, fields := range records {
+		if err := c.SetMetadataFlat(id, fields); err != nil {
+			t.Fatalf("seed %s: %v", id, err)
+		}
+	}
+}
+
+// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
+func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
+	for _, v := range values {
+		if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" {
+			return true
+		}
+	}
+	prefix := strings.ToLower(prop) + "/"
+	for k, val := range fields {
+		if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) {
+			for _, v := range values {
+				if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" {
+					return true
+				}
+			}
+		}
+	}
+	fv, ok := fields[prop]
+	if !ok {
+		return false
+	}
+	for _, piece := range strings.Split(fv, ",") {
+		piece = strings.TrimSpace(piece)
+		for _, v := range values {
+			if exact && strings.EqualFold(piece, v) {
+				return true
+			}
+			if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string {
+	t.Helper()
+	hits, err := c.Search(query, filters, 100, match)
+	if err != nil {
+		t.Fatalf("search: %v", err)
+	}
+	ids := make([]string, 0, len(hits))
+	for _, h := range hits {
+		ids = append(ids, h.HashID)
+	}
+	return ids
+}
+
+func textMatch(query string) func(string, map[string]string) bool {
+	tokens := strings.Fields(strings.ToLower(query))
+	return func(id string, fields map[string]string) bool {
+		hay := Haystack(fields)
+		for _, tok := range tokens {
+			if !strings.Contains(hay, tok) {
+				return strings.Contains(strings.ToLower(id), strings.ToLower(query))
+			}
+		}
+		return true
+	}
+}
+
+func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool {
+	return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) }
+}
+
+func equalIDs(t *testing.T, got []string, want ...string) {
+	t.Helper()
+	sort.Strings(want)
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestSearchFilterCandidates(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"},
+		"R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"},
+		"R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"},
+		"R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"},
+	})
+
+	cases := []struct {
+		name   string
+		prop   string
+		exact  bool
+		values []string
+		want   []string
+	}{
+		{"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}},
+		{"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}},
+		{"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}},
+		{"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}},
+		{"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}},
+		{"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}},
+		{"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}},
+		{"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}},
+		{"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...))
+			equalIDs(t, got, tc.want...)
+		})
+	}
+}
+
+func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"R1": {"genres/Action": "true", "fileType": "video"},
+		"R2": {"genres/Action": "true", "fileType": "audio"},
+	})
+	match := func(_ string, f map[string]string) bool {
+		return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video")
+	}
+	got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match)
+	equalIDs(t, got, "R1")
+}
+
+func TestSearchFreeTextCandidates(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"},
+		"01KBBB": {"title": "Dune: Part Two"},
+		"01KCCC": {"title": "進撃の巨人"},
+		"01KDDD": {"title": "3%", "titles/eng/3 percent": "true"},
+		"01KEEE": {"title": "AND -x a:b * \"quoted\""},
+		"01KFFF": {"title": "Unrelated"},
+	})
+	cases := []struct {
+		query string
+		want  []string
+	}{
+		{"naruto", []string{"01KAAA"}},
+		{"NARUTO s01e02", []string{"01KAAA"}},
+		{"dune:", []string{"01KBBB"}},
+		{"巨人", []string{"01KCCC"}},        // 2 runes: below trigram length
+		{"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set
+		{"3%", []string{"01KDDD"}},        // LIKE metacharacter in a short token
+		{"AND", []string{"01KEEE"}},       // FTS operator as a literal
+		{"-x", []string{"01KEEE"}},
+		{"a:b", []string{"01KEEE"}},
+		{"*", []string{"01KEEE"}},
+		{`"quoted"`, []string{"01KEEE"}},
+		{"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring
+	}
+	for _, tc := range cases {
+		t.Run(tc.query, func(t *testing.T) {
+			got := search(t, c, tc.query, nil, textMatch(tc.query))
+			equalIDs(t, got, tc.want...)
+		})
+	}
+}
+
+// A rid token and a title token must not match together: the rid is in the
+// FTS row only as a candidate path for the "whole query is a hashID substring"
+// rule, and verification checks the two separately.
+func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}})
+	if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 {
+		t.Fatalf("rid + title token matched: %v", got)
+	}
+}
+
+func TestSearchFollowsWritesAndDeletes(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
+	if err := c.SetProperty("R1", "title", "After"); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
+	equalIDs(t, search(t, c, "before", nil, textMatch("before")))
+	if _, err := c.DeleteMetadata("R1"); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "after", nil, textMatch("after")))
+	equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
+}
+
+func TestSearchPagesPastRejectedCandidates(t *testing.T) {
+	c := newTestClient(t)
+	records := map[string]map[string]string{}
+	for i := 0; i < 450; i++ {
+		id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3)
+		records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"}
+	}
+	records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"}
+	seed(t, c, records)
+	got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}},
+		func(_ string, f map[string]string) bool { return f["keep"] == "yes" })
+	equalIDs(t, got, "RZZZZ")
+}
+
+// The filter candidates only scale if SQLite actually uses the partial
+// expression index — which it does only when the query repeats the index's
+// expression and WHERE clause as literals.
+func TestSearchFilterUsesFieldIndex(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}})
+	q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}})
+	rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	var plan []string
+	for rows.Next() {
+		var id, parent, notused int
+		var detail string
+		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
+			t.Fatal(err)
+		}
+		plan = append(plan, detail)
+	}
+	joined := strings.Join(plan, " | ")
+	if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") {
+		t.Fatalf("filter candidates do not use kv_field_v: %s", joined)
+	}
+}
+
+func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) {
+	if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" {
+		t.Fatalf("got %q %q %v", h, f, ok)
+	}
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}})
+	got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video"))
+	equalIDs(t, got, "bafkreiabcdefghijklmnop")
+}
+
+func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}})
+	if err := c.update(func(tx *kv.Tx) error {
+		if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
+			return err
+		}
+		_, err := tx.DeletePrefix(ftsNS, 0)
+		return err
+	}); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")))
+	n, err := c.RebuildSearchIfEmpty()
+	if err != nil || n != 2 {
+		t.Fatalf("rebuilt %d (err %v), want 2", n, err)
+	}
+	equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1")
+	if n, _ := c.RebuildSearchIfEmpty(); n != 0 {
+		t.Fatalf("second rebuild ran again (%d)", n)
+	}
+}
+
+// Text plus a broad filter: the selective part drives and the filter becomes a
+// per-candidate probe over the record's own key range — never a full
+// materialisation of the filter's result set.
+func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
+	c := newTestClient(t)
+	recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}}
+	for i := 1; i < 300; i++ {
+		recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"}
+	}
+	seed(t, c, recs)
+	filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
+	driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 {
+		t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes))
+	}
+	q, pargs := probeSQL(probes)
+	args := append([]any{`["R0000"]`}, pargs...)
+	rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	var plan []string
+	for rows.Next() {
+		var id, parent, notused int
+		var detail string
+		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
+			t.Fatal(err)
+		}
+		plan = append(plan, detail)
+	}
+	joined := strings.Join(plan, " | ")
+	if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
+		t.Fatalf("filter probe is not a primary-key range read: %s", joined)
+	}
+	equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool {
+		return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
+	}), "R0000")
+}

internal/storage/stats.go

@@ -0,0 +1,45 @@
+package storage
+
+import (
+	"time"
+)
+
+// CountRoots returns the number of metadata roots in the root index — one
+// range count over idx:roots/, never the list itself.
+func (c *Client) CountRoots() (int64, error) {
+	s, err := c.kv()
+	if err != nil {
+		return 0, err
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	return s.Count(ctx, rootsNS)
+}
+
+// keyCountTTL bounds how stale CountKeys may be. Counting every row is a full
+// table scan (~a second at millions of keys), and the only reader is a polled
+// dashboard tile — it needs a number, not a fresh one every few seconds.
+const keyCountTTL = 60 * time.Second
+
+// CountKeys returns the number of rows in the database — every key, not just
+// records. It is the denominator for "how much of the store is records": the
+// flat per-field keys (file:{root}/{field}) dominate, which is expected.
+func (c *Client) CountKeys() (int64, error) {
+	c.keysMu.Lock()
+	defer c.keysMu.Unlock()
+	if !c.keysAt.IsZero() && time.Since(c.keysAt) < keyCountTTL {
+		return c.keysCount, nil
+	}
+	s, err := c.kv()
+	if err != nil {
+		return 0, err
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	n, err := s.CountAll(ctx)
+	if err != nil {
+		return 0, err
+	}
+	c.keysCount, c.keysAt = n, time.Now()
+	return n, nil
+}

internal/storage/udl.go

@@ -12,25 +12,31 @@ package storage
 // strictly exceeds the stored one (a per-cell high-water-mark, replacing the
 // op-log merge that used to live in meta-watch's store.rs).
 //
-// Redis model (all under the literal "udl:" prefix, mirroring the "file:" one):
+// Key model (all under the literal "udl:" prefix, mirroring the "file:" one):
 //
-//	udl:rec:<uid>/<cid>/<key>       HASH { record, version, ts, value? }
-//	udl:idx:user:<uid>:key:<key>    SET of <cid>   -> active_for_user_key (My List, Continue Watching)
-//	udl:idx:user:<uid>:cid:<cid>    SET of <key>   -> list_for_user_cid
-//	udl:idx:cid:<cid>:key:<key>     SET of <uid>   -> aggregate
-//	udl:idx:cid:<cid>:users         SET of <uid>   -> uids_for_cid
+//	udl:rec:<uid>/<cid>/<key>       JSON { record, version, ts, value? }
+//	udl:idx:user:<uid>:key:<key>    set of <cid>   -> active_for_user_key (My List, Continue Watching)
+//	udl:idx:user:<uid>:cid:<cid>    set of <key>   -> list_for_user_cid
+//	udl:idx:user:<uid>:cids         set of <cid>   -> whole-profile export (recovery)
+//	udl:idx:cid:<cid>:key:<key>     set of <uid>   -> aggregate
+//	udl:idx:cid:<cid>:users         set of <uid>   -> uids_for_cid
 //
-// The `value` hash field is the plaintext JSON scalar for PUBLIC keys only
+// A "set" is a key-set: one key `<set>\x1f<member>` = "" per member, read as
+// the key range under `<set>\x1f`. The separator is a control character
+// because UDL keys are not validated — with `/` the members of
+// `…:key:a/b` would show up as members of `…:key:a`.
+//
+// The `value` field is the plaintext JSON scalar for PUBLIC keys only
 // (like/rating), used for aggregation; it is absent for private-tier keys
 // (whose real value rides encrypted inside `record`) and for tombstones.
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"strconv"
-	"time"
+	"sort"
 
-	"github.com/redis/go-redis/v9"
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 // UDLCell is a decoded (uid, cid, key) record cell.
@@ -66,151 +72,208 @@ func udlIdxUserCid(uid, cid string) string { return fmt.Sprintf("udl:idx:user:%s
 func udlIdxCidKey(cid, key string) string  { return fmt.Sprintf("udl:idx:cid:%s:key:%s", cid, key) }
 func udlIdxCidUsers(cid string) string     { return fmt.Sprintf("udl:idx:cid:%s:users", cid) }
 
-// udlUpsertScript performs the version-gated upsert + index maintenance
-// atomically. Accepts (returns 1) only when the incoming version strictly
-// exceeds the stored one; otherwise returns 0 (stale).
+// udlIdxUserCids is the set of every cid a uid has ANY record for — the one
+// index that makes "export this whole profile" a set read instead of a
+// keyspace scan.
 //
-//	KEYS: 1=cell 2=idx:user:key 3=idx:user:cid 4=idx:cid:key 5=idx:cid:users
-//	ARGV: 1=version 2=ts 3=record 4=value 5=hasValue("1"/"0")
-//	      6=cid 7=key 8=uid 9=tombstone("1"/"0")
-//
-// A tombstone still writes the cell — that is the CRDT state, and dropping it
-// would let an older write resurrect the value — but it SREMs the cid from the
-// user+key index. That index is what `active_for_user_key` returns, so leaving
-// deleted cids in it means every My List / Continue Watching read carries every
-// title ever un-liked or finished, forever. The other three indexes are left
-// alone: they answer "who has an opinion about this cid", where a tombstone is
-// still an answer.
-var udlUpsertScript = redis.NewScript(`
-local stored = redis.call('HGET', KEYS[1], 'version')
-if stored and tonumber(stored) >= tonumber(ARGV[1]) then
-  return 0
-end
-redis.call('HSET', KEYS[1], 'version', ARGV[1], 'ts', ARGV[2], 'record', ARGV[3])
-if ARGV[5] == '1' then
-  redis.call('HSET', KEYS[1], 'value', ARGV[4])
-else
-  redis.call('HDEL', KEYS[1], 'value')
-end
-if ARGV[9] == '1' then
-  redis.call('SREM', KEYS[2], ARGV[6])
-else
-  redis.call('SADD', KEYS[2], ARGV[6])
-end
-redis.call('SADD', KEYS[3], ARGV[7])
-redis.call('SADD', KEYS[4], ARGV[8])
-redis.call('SADD', KEYS[5], ARGV[8])
-return 1
-`)
+// The other four indexes each answer a question with one coordinate already
+// known (a key, a cid). Cross-instance recovery knows neither: it has a uid and
+// nothing else. Without this set the only ways to enumerate a profile are to
+// iterate the hardcoded key list — which silently misses any key added later —
+// or to SCAN `udl:rec:<uid>/*`, which is the full-keyspace-read pathology that
+// already made `/api/udl/users/stats` admin-only.
+func udlIdxUserCids(uid string) string { return fmt.Sprintf("udl:idx:user:%s:cids", uid) }
+
+// udlSetSep separates a set key from a member (see the key model above).
+const udlSetSep = "\x1f"
+
+// udlCellJSON is a cell's stored form. Value is a pointer so an absent value
+// (private-tier key, tombstone) stays distinguishable from an empty one.
+type udlCellJSON struct {
+	Record  string  `json:"record"`
+	Version int64   `json:"version"`
+	Ts      int64   `json:"ts"`
+	Value   *string `json:"value,omitempty"`
+}
+
+func decodeUDLCell(raw string) (UDLCell, error) {
+	var j udlCellJSON
+	if err := json.Unmarshal([]byte(raw), &j); err != nil {
+		return UDLCell{}, err
+	}
+	cell := UDLCell{Record: j.Record, Version: j.Version, Ts: j.Ts}
+	if j.Value != nil {
+		cell.Value, cell.HasValue = *j.Value, true
+	}
+	return cell, nil
+}
+
+// udlMembers lists a key-set's members, sorted.
+func udlMembers(ctx context.Context, r kv.Reader, set string) ([]string, error) {
+	prefix := set + udlSetSep
+	rows, err := r.Range(ctx, prefix, "", 0)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]string, 0, len(rows))
+	for _, row := range rows {
+		out = append(out, row.K[len(prefix):])
+	}
+	return out, nil
+}
+
+func udlAdd(tx *kv.Tx, set, member string) error {
+	_, err := tx.PutIfAbsent(set+udlSetSep+member, "")
+	return err
+}
+
+func udlRem(tx *kv.Tx, set, member string) error {
+	_, err := tx.Delete(set + udlSetSep + member)
+	return err
+}
+
+// readUDLCell reads one cell. exists is false when it was never written.
+func readUDLCell(ctx context.Context, r kv.Reader, recKey string) (UDLCell, bool, error) {
+	raw, ok, err := r.Get(ctx, recKey)
+	if err != nil || !ok {
+		return UDLCell{}, false, err
+	}
+	cell, err := decodeUDLCell(raw)
+	if err != nil {
+		return UDLCell{}, false, fmt.Errorf("decode %s: %w", recKey, err)
+	}
+	return cell, true, nil
+}
+
+// readUDLCells reads many cells in one multi-key read; absent cells are
+// omitted from the map.
+func readUDLCells(ctx context.Context, r kv.Reader, recKeys []string) (map[string]UDLCell, error) {
+	raws, err := r.MGet(ctx, recKeys)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]UDLCell, len(raws))
+	for k, raw := range raws {
+		cell, err := decodeUDLCell(raw)
+		if err != nil {
+			return nil, fmt.Errorf("decode %s: %w", k, err)
+		}
+		out[k] = cell
+	}
+	return out, nil
+}
 
 // UDLUpsertIfNewer stores a record cell iff version strictly exceeds the stored
-// one. publicValue/hasValue carry the plaintext JSON scalar for public keys
-// (used by aggregation); pass hasValue=false for private-tier keys and
-// tombstones. Set tombstone=true when the record deletes the cell, which also
-// prunes the cid from the user+key index. Returns accepted=false when the write
-// is stale.
+// one, and maintains the indexes — one transaction, so the gate and the write
+// cannot interleave with a concurrent upsert of the same cell.
+// publicValue/hasValue carry the plaintext JSON scalar for public keys (used by
+// aggregation); pass hasValue=false for private-tier keys and tombstones. Set
+// tombstone=true when the record deletes the cell.
+//
+// A tombstone still writes the cell — that is the CRDT state, and dropping it
+// would let an older write resurrect the value — but it removes the cid from
+// the user+key index. That index is what `active_for_user_key` returns, so
+// leaving deleted cids in it means every My List / Continue Watching read
+// carries every title ever un-liked or finished, forever. The other four
+// indexes are left alone: they answer "who has an opinion about this cid",
+// where a tombstone is still an answer.
+//
+// ⚠ `idx:user:cids` in particular must NEVER be pruned on a tombstone, even
+// though it looks symmetric with the user+key index. It is the export index: a
+// cid dropped from it is a cid whose tombstone never travels, and a tombstone
+// that does not travel is exactly how an un-liked title comes back from the
+// dead on the other box — the peer still holds the older `like` at a lower
+// version, and with no tombstone to beat it, nothing ever supersedes it.
+//
+// Returns accepted=false when the write is stale.
 func (c *Client) UDLUpsertIfNewer(uid, cid, key string, version, ts int64, record, publicValue string, hasValue, tombstone bool) (bool, error) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if c.client == nil {
-		return false, fmt.Errorf("not connected")
+	accepted := false
+	err := c.update(func(tx *kv.Tx) error {
+		recKey := udlRecKey(uid, cid, key)
+		stored, ok, err := readUDLCell(tx.Context(), tx, recKey)
+		if err != nil {
+			return err
 		}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	keys := []string{
-		c.buildKey(udlRecKey(uid, cid, key)),
-		c.buildKey(udlIdxUserKey(uid, key)),
-		c.buildKey(udlIdxUserCid(uid, cid)),
-		c.buildKey(udlIdxCidKey(cid, key)),
-		c.buildKey(udlIdxCidUsers(cid)),
+		if ok && stored.Version >= version {
+			return nil
 		}
-	hv := "0"
+		cell := udlCellJSON{Record: record, Version: version, Ts: ts}
 		if hasValue {
-		hv = "1"
+			v := publicValue
+			cell.Value = &v
+		}
+		body, err := json.Marshal(cell)
+		if err != nil {
+			return err
+		}
+		if err := tx.Put(recKey, string(body)); err != nil {
+			return err
 		}
-	tomb := "0"
 		if tombstone {
-		tomb = "1"
+			err = udlRem(tx, udlIdxUserKey(uid, key), cid)
+		} else {
+			err = udlAdd(tx, udlIdxUserKey(uid, key), cid)
 		}
-	res, err := udlUpsertScript.Run(ctx, c.client, keys, version, ts, record, publicValue, hv, cid, key, uid, tomb).Result()
+		if err != nil {
+			return err
+		}
+		for _, add := range [][2]string{
+			{udlIdxUserCid(uid, cid), key},
+			{udlIdxCidKey(cid, key), uid},
+			{udlIdxCidUsers(cid), uid},
+			{udlIdxUserCids(uid), cid},
+		} {
+			if err := udlAdd(tx, add[0], add[1]); err != nil {
+				return err
+			}
+		}
+		accepted = true
+		return nil
+	})
 	if err != nil {
 		return false, fmt.Errorf("udl upsert failed: %w", err)
 	}
-	n, _ := res.(int64)
-	return n == 1, nil
+	return accepted, nil
 }
 
 // UDLGetCell returns the record cell for (uid, cid, key). exists is false when
 // no record has ever been written for the cell.
 func (c *Client) UDLGetCell(uid, cid, key string) (UDLCell, bool, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return UDLCell{}, false, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	return c.udlReadCell(ctx, udlRecKey(uid, cid, key))
-}
-
-// udlReadCell HMGETs a cell hash. Caller holds the lock. exists is false when
-// the version field is absent (i.e. the hash does not exist).
-func (c *Client) udlReadCell(ctx context.Context, recKey string) (UDLCell, bool, error) {
-	vals, err := c.client.HMGet(ctx, c.buildKey(recKey), "record", "version", "ts", "value").Result()
+	s, err := c.kv()
 	if err != nil {
-		return UDLCell{}, false, fmt.Errorf("hmget failed: %w", err)
+		return UDLCell{}, false, err
 	}
-	// vals: [record, version, ts, value] — nil for absent fields.
-	if len(vals) < 4 || vals[1] == nil {
-		return UDLCell{}, false, nil
-	}
-	cell := UDLCell{}
-	if s, ok := vals[0].(string); ok {
-		cell.Record = s
-	}
-	if s, ok := vals[1].(string); ok {
-		cell.Version, _ = strconv.ParseInt(s, 10, 64)
-	}
-	if s, ok := vals[2].(string); ok {
-		cell.Ts, _ = strconv.ParseInt(s, 10, 64)
-	}
-	if s, ok := vals[3].(string); ok {
-		cell.Value = s
-		cell.HasValue = true
-	}
-	return cell, true, nil
+	ctx, cancel := opCtx()
+	defer cancel()
+	return readUDLCell(ctx, s, udlRecKey(uid, cid, key))
 }
 
 // UDLListUserKey returns every cell for (uid, key) across all cids — the
 // projection behind active_for_user_key (My List, Continue Watching). Includes
 // tombstones; the caller decodes each record and filters.
 func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	cids, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserKey(uid, key))).Result()
-	if err != nil {
-		return nil, fmt.Errorf("smembers failed: %w", err)
-	}
-	out := make([]UDLUserKeyEntry, 0, len(cids))
-	for _, cid := range cids {
-		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
+	ctx, cancel := opCtx()
+	defer cancel()
+
+	cids, err := udlMembers(ctx, s, udlIdxUserKey(uid, key))
+	if err != nil {
+		return nil, err
+	}
+	recKeys := make([]string, len(cids))
+	for i, cid := range cids {
+		recKeys[i] = udlRecKey(uid, cid, key)
+	}
+	cells, err := readUDLCells(ctx, s, recKeys)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]UDLUserKeyEntry, 0, len(cids))
+	for i, cid := range cids {
+		cell, ok := cells[recKeys[i]]
 		if !ok {
 			continue
 		}
@@ -222,26 +285,28 @@ func (c *Client) UDLListUserKey(uid, key string) ([]UDLUserKeyEntry, error) {
 // UDLListUserCid returns every cell for (uid, cid) across all keys — the
 // projection behind list_for_user_cid.
 func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-
-	keys, err := c.client.SMembers(ctx, c.buildKey(udlIdxUserCid(uid, cid))).Result()
-	if err != nil {
-		return nil, fmt.Errorf("smembers failed: %w", err)
-	}
-	out := make([]UDLUserCidEntry, 0, len(keys))
-	for _, key := range keys {
-		cell, ok, err := c.udlReadCell(ctx, udlRecKey(uid, cid, key))
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
+	ctx, cancel := opCtx()
+	defer cancel()
+
+	keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
+	if err != nil {
+		return nil, err
+	}
+	recKeys := make([]string, len(keys))
+	for i, key := range keys {
+		recKeys[i] = udlRecKey(uid, cid, key)
+	}
+	cells, err := readUDLCells(ctx, s, recKeys)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]UDLUserCidEntry, 0, len(keys))
+	for i, key := range keys {
+		cell, ok := cells[recKeys[i]]
 		if !ok {
 			continue
 		}
@@ -250,53 +315,128 @@ func (c *Client) UDLListUserCid(uid, cid string) ([]UDLUserCidEntry, error) {
 	return out, nil
 }
 
-// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
-func (c *Client) UDLCidUsers(cid string) ([]string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
+// UDLUserAllEntry is one row of a whole-profile export.
+type UDLUserAllEntry struct {
+	Cid     string `json:"cid"`
+	Key     string `json:"key"`
+	Record  string `json:"record"`
+	Version int64  `json:"version"`
+	Ts      int64  `json:"ts"`
+}
 
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
+// UDLAllForUser returns every cell a uid owns, for cross-instance recovery.
+//
+// Ordered by cid, starting strictly after `afterCid`. A cid's keys are never
+// split across a page: the page ends on a cid boundary once `limit` rows have
+// been emitted, and `nextCid` is the last cid included — pass it back as
+// `afterCid` to continue. `nextCid` is empty when the export is complete.
+//
+// Tombstones are included. They are the CRDT state that stops a deleted value
+// resurrecting on the box that receives this export, so omitting them would
+// make recovery actively wrong rather than merely incomplete.
+func (c *Client) UDLAllForUser(uid, afterCid string, limit int) ([]UDLUserAllEntry, string, error) {
+	s, err := c.kv()
+	if err != nil {
+		return nil, "", err
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	if limit <= 0 {
+		limit = 1000
+	}
+	ctx, cancel := opCtx()
 	defer cancel()
 
-	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidUsers(cid))).Result()
+	cids, err := udlMembers(ctx, s, udlIdxUserCids(uid))
 	if err != nil {
-		return nil, fmt.Errorf("smembers failed: %w", err)
+		return nil, "", err
 	}
-	return uids, nil
+	sort.Strings(cids)
+
+	out := make([]UDLUserAllEntry, 0, limit)
+	nextCid := ""
+	for _, cid := range cids {
+		if afterCid != "" && cid <= afterCid {
+			continue
+		}
+		if len(out) >= limit {
+			nextCid = out[len(out)-1].Cid
+			break
+		}
+		keys, err := udlMembers(ctx, s, udlIdxUserCid(uid, cid))
+		if err != nil {
+			return nil, "", err
+		}
+		for _, key := range keys {
+			cell, ok, err := readUDLCell(ctx, s, udlRecKey(uid, cid, key))
+			if err != nil {
+				return nil, "", err
+			}
+			if !ok {
+				continue
+			}
+			out = append(out, UDLUserAllEntry{
+				Cid: cid, Key: key, Record: cell.Record, Version: cell.Version, Ts: cell.Ts,
+			})
+		}
+	}
+	return out, nextCid, nil
+}
+
+// UDLCountUserCids is the number of distinct cids a uid holds records for.
+//
+// One index range count, which is what makes it usable as a per-push quota
+// check on the gateway mirror — a quota that cost a keyspace walk would be a
+// denial-of-service in its own right. Counts cids rather than cells because
+// that is what the index holds; a profile is bounded by the titles it has
+// touched, not by how many keys it has per title.
+func (c *Client) UDLCountUserCids(uid string) (int64, error) {
+	s, err := c.kv()
+	if err != nil {
+		return 0, err
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	return s.Count(ctx, udlIdxUserCids(uid)+udlSetSep)
+}
+
+// UDLCidUsers returns every uid that has any record for cid — uids_for_cid.
+func (c *Client) UDLCidUsers(cid string) ([]string, error) {
+	s, err := c.kv()
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := opCtx()
+	defer cancel()
+	return udlMembers(ctx, s, udlIdxCidUsers(cid))
 }
 
 // UDLPublicValues returns the live plaintext public JSON scalar values for
 // (cid, key) across all uids — the input to aggregation. Cells without a value
-// field (private-tier or tombstoned) are skipped.
+// (private-tier or tombstoned) are skipped.
 func (c *Client) UDLPublicValues(cid, key string) ([]string, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
+	s, err := c.kv()
+	if err != nil {
+		return nil, err
 	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := opCtx()
 	defer cancel()
 
-	uids, err := c.client.SMembers(ctx, c.buildKey(udlIdxCidKey(cid, key))).Result()
+	uids, err := udlMembers(ctx, s, udlIdxCidKey(cid, key))
 	if err != nil {
-		return nil, fmt.Errorf("smembers failed: %w", err)
+		return nil, err
+	}
+	recKeys := make([]string, len(uids))
+	for i, uid := range uids {
+		recKeys[i] = udlRecKey(uid, cid, key)
+	}
+	cells, err := readUDLCells(ctx, s, recKeys)
+	if err != nil {
+		return nil, err
 	}
 	out := make([]string, 0, len(uids))
-	for _, uid := range uids {
-		v, err := c.client.HGet(ctx, c.buildKey(udlRecKey(uid, cid, key)), "value").Result()
-		if err == redis.Nil {
-			continue // private-tier or tombstoned
+	for _, k := range recKeys {
+		if cell, ok := cells[k]; ok && cell.HasValue {
+			out = append(out, cell.Value)
 		}
-		if err != nil {
-			return nil, fmt.Errorf("hget failed: %w", err)
-		}
-		out = append(out, v)
 	}
 	return out, nil
 }

internal/storage/udl_admin.go

@@ -3,22 +3,16 @@ package storage
 // Account-scoped UDL administration: what a uid holds, and removing all of it.
 //
 // Both operations enumerate the same thing — every `udl:rec:<uid>/<cid>/<key>`
-// cell — because there is no per-user cid index to consult. The Redis model
-// (see udl.go) indexes user+key → cids and user+cid → keys, which answer
-// "what is in My List", not "everything this account ever wrote". Adding a
-// third index would put a write on every UDL upsert to serve two admin calls,
-// so these SCAN instead and accept being O(keyspace).
-//
-// That cost is why UDLAllUserStats exists as a bulk call rather than a per-uid
-// one: the accounts page needs a number for every account, and MATCH filters
-// server-side without indexing, so N per-uid calls would walk the keyspace N
-// times. One walk buckets them all.
+// cell. Keys are ordered, so one account's cells are the key range under
+// `udl:rec:<uid>/`, and every account's are the range under `udl:rec:`.
+// UDLAllUserStats stays a bulk call (one walk buckets every account) because
+// the accounts page needs a number for each.
 
 import (
-	"context"
 	"fmt"
 	"strings"
-	"time"
+
+	"github.com/metazla/meta-core/internal/kv"
 )
 
 // UDLUserStats counts what one account holds. Records is the number of cells,
@@ -30,11 +24,8 @@ type UDLUserStats struct {
 	Keys    int `json:"keys"`
 }
 
-// udlScanBatch is the SCAN COUNT hint, matching the rest of this package.
-const udlScanBatch = 1000
-
-// safeUIDForPattern rejects anything that would change the meaning of a SCAN
-// glob or reach across the key structure. meta-core's uids are alphanumeric
+// safeUIDForPattern rejects anything that would reach across the key
+// structure (a `/` would let a uid prefix-match another account's range). meta-core's uids are alphanumeric
 // (identity.validUID), so this only ever fires on a hostile or malformed one —
 // but it fires here, in the layer that builds the pattern, rather than relying
 // on every caller having validated first.
@@ -62,184 +53,127 @@ func splitUDLRecSuffix(suffix string) (uid, cid, key string, ok bool) {
 	return parts[0], parts[1], parts[2], true
 }
 
-// scanKeys walks the keyspace for pattern and returns every match. Caller holds
-// the lock.
-//
-// SCAN can return the same key more than once across cursor iterations (it
-// guarantees no misses, not no duplicates), so callers that count must
-// de-duplicate — both callers below collect into maps or sets for that reason.
-func (c *Client) scanKeys(ctx context.Context, pattern string) ([]string, error) {
-	var (
-		out    []string
-		cursor uint64
-	)
-	for {
-		keys, next, err := c.client.Scan(ctx, cursor, pattern, udlScanBatch).Result()
-		if err != nil {
-			return nil, fmt.Errorf("scan %q: %w", pattern, err)
-		}
-		out = append(out, keys...)
-		cursor = next
-		if cursor == 0 {
-			return out, nil
-		}
-	}
-}
-
-// UDLUserCells returns every (cid, key) pair this account holds a cell for.
-// De-duplicated, since SCAN may repeat a key.
+// UDLUserCells returns every (cid, key) pair this account holds a cell for,
+// keyed by the "cid/key" tail with the cid as value (the purge needs them
+// grouped).
 func (c *Client) UDLUserCells(uid string) (map[string]string, error) {
-	// Keyed by the full "cid/key" tail so the pair stays a pair; the value is
-	// the cid, which the purge needs grouped.
 	if err := safeUIDForPattern(uid); err != nil {
 		return nil, err
 	}
-
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
-	defer cancel()
-
-	prefix := c.buildKey("udl:rec:")
-	keys, err := c.scanKeys(ctx, prefix+uid+"/*")
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
-	out := make(map[string]string, len(keys))
-	for _, full := range keys {
-		gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
-		// The glob can only match this uid, but a uid that is a prefix of
-		// another would still be caught by the '/' in the pattern; check anyway
-		// so a malformed key can never be attributed to the wrong account.
+	ctx, cancel := opCtx()
+	defer cancel()
+
+	out := make(map[string]string)
+	err = s.ScanPrefix(ctx, "udl:rec:"+uid+"/", readerPage, func(rows []kv.KV) error {
+		for _, row := range rows {
+			gotUID, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
 			if !ok || gotUID != uid {
 				continue
 			}
 			out[cid+"/"+key] = cid
 		}
-	return out, nil
+		return nil
+	})
+	return out, err
 }
 
-// UDLAllUserStats counts every account's cells in one keyspace walk.
+// UDLAllUserStats counts every account's cells in one walk.
 //
 // Returns only accounts that actually hold records; a uid absent from the map
 // holds none. Callers render a zero for those rather than treating it as an
 // error — an account created a minute ago legitimately has nothing.
 func (c *Client) UDLAllUserStats() (map[string]UDLUserStats, error) {
-	c.mu.RLock()
-	defer c.mu.RUnlock()
-	if c.client == nil {
-		return nil, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-
-	prefix := c.buildKey("udl:rec:")
-	keys, err := c.scanKeys(ctx, prefix+"*")
+	s, err := c.kv()
 	if err != nil {
 		return nil, err
 	}
+	ctx, cancel := opCtx()
+	defer cancel()
 
-	// Sets rather than counters: SCAN may hand back the same key twice, which
-	// would otherwise inflate every number on the page.
 	type acc struct {
-		cells map[string]struct{}
+		cells int
 		cids  map[string]struct{}
 		keys  map[string]struct{}
 	}
 	byUID := make(map[string]*acc)
-	for _, full := range keys {
-		uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(full, prefix))
+	err = s.ScanPrefix(ctx, "udl:rec:", readerPage, func(rows []kv.KV) error {
+		for _, row := range rows {
+			uid, cid, key, ok := splitUDLRecSuffix(strings.TrimPrefix(row.K, "udl:rec:"))
 			if !ok {
 				continue
 			}
 			a := byUID[uid]
 			if a == nil {
-			a = &acc{
-				cells: make(map[string]struct{}),
-				cids:  make(map[string]struct{}),
-				keys:  make(map[string]struct{}),
-			}
+				a = &acc{cids: make(map[string]struct{}), keys: make(map[string]struct{})}
 				byUID[uid] = a
 			}
-		a.cells[cid+"/"+key] = struct{}{}
+			a.cells++
 			a.cids[cid] = struct{}{}
 			a.keys[key] = struct{}{}
 		}
-
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
 	out := make(map[string]UDLUserStats, len(byUID))
 	for uid, a := range byUID {
-		out[uid] = UDLUserStats{Records: len(a.cells), Cids: len(a.cids), Keys: len(a.keys)}
+		out[uid] = UDLUserStats{Records: a.cells, Cids: len(a.cids), Keys: len(a.keys)}
 	}
 	return out, nil
 }
 
 // UDLPurgeUser removes every trace of one account from the User Data Layer:
-// its cells, its two user-scoped indexes, and its membership in the shared
+// its cells, its three user-scoped indexes, and its membership in the shared
 // cid-scoped indexes. Returns what was removed.
 //
 // The shared indexes are the part that is easy to miss and expensive to leave.
 // `udl:idx:cid:<cid>:key:<key>` and `udl:idx:cid:<cid>:users` are keyed by cid,
 // not by uid, so a purge that only deleted `udl:rec:<uid>/*` would leave this
-// uid listed as an opinion-holder on every title it ever touched. Aggregation
-// then HGETs a cell that no longer exists — UDLPublicValues skips it, so the
-// rating average stays right, but uids_for_cid keeps naming a deleted account
-// forever and every aggregate read pays for the lookup.
+// uid listed as an opinion-holder on every title it ever touched.
 //
 // Idempotent: purging an account with nothing stored is a no-op returning
-// zeroes, so a retry after a partial failure is safe.
+// zeroes, so a retry is safe.
 func (c *Client) UDLPurgeUser(uid string) (UDLUserStats, error) {
 	cells, err := c.UDLUserCells(uid)
 	if err != nil {
 		return UDLUserStats{}, err
 	}
 
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	if c.client == nil {
-		return UDLUserStats{}, fmt.Errorf("not connected")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-
 	stats := UDLUserStats{Records: len(cells)}
 	distinctCids := make(map[string]struct{})
 	distinctKeys := make(map[string]struct{})
-
-	pipe := c.client.Pipeline()
+	err = c.update(func(tx *kv.Tx) error {
 		for cidKey, cid := range cells {
 			key := strings.TrimPrefix(cidKey, cid+"/")
 			distinctCids[cid] = struct{}{}
 			distinctKeys[key] = struct{}{}
-		pipe.Del(ctx, c.buildKey(udlRecKey(uid, cid, key)))
-		pipe.SRem(ctx, c.buildKey(udlIdxCidKey(cid, key)), uid)
+			if _, err := tx.Delete(udlRecKey(uid, cid, key)); err != nil {
+				return err
+			}
+			if err := udlRem(tx, udlIdxCidKey(cid, key), uid); err != nil {
+				return err
+			}
 		}
 		for cid := range distinctCids {
-		pipe.SRem(ctx, c.buildKey(udlIdxCidUsers(cid)), uid)
+			if err := udlRem(tx, udlIdxCidUsers(cid), uid); err != nil {
+				return err
+			}
+		}
+		// The three user-scoped indexes (`:key:*`, `:cid:*`, `:cids`) are one
+		// key range, dropped whole — which also catches index entries whose
+		// cell is already gone (the state a half-finished purge leaves).
+		_, err := tx.DeletePrefix("udl:idx:user:"+uid+":", 0)
+		return err
+	})
+	if err != nil {
+		return UDLUserStats{}, fmt.Errorf("purge: %w", err)
 	}
 	stats.Cids = len(distinctCids)
 	stats.Keys = len(distinctKeys)
-
-	// The two user-scoped indexes are enumerable directly, and are dropped
-	// whole rather than member-by-member. Scanning them separately also catches
-	// index entries whose cell is already gone — exactly the state a previously
-	// half-finished purge leaves behind.
-	idxPrefix := c.buildKey("udl:idx:user:") + uid + ":"
-	idxKeys, err := c.scanKeys(ctx, idxPrefix+"*")
-	if err != nil {
-		return UDLUserStats{}, err
-	}
-	for _, k := range idxKeys {
-		pipe.Del(ctx, k)
-	}
-
-	if _, err := pipe.Exec(ctx); err != nil {
-		return UDLUserStats{}, fmt.Errorf("purge pipeline: %w", err)
-	}
 	return stats, nil
 }

internal/storage/udl_admin_test.go

@@ -20,7 +20,7 @@ func seedCell(t *testing.T, c *Client, uid, cid, key string) {
 // forever — and the rows are unreachable through any API, so nothing will ever
 // clean them up.
 func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "cidOne", "like")
 	seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -63,7 +63,7 @@ func TestUDLPurgeUser_RemovesCellsAndBothIndexFamilies(t *testing.T) {
 // both have an opinion about — where the two share an index set and a careless
 // DEL of the whole key would take the survivor's membership with it.
 func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "shared", "rating")
 	seedCell(t, c, "zBob", "shared", "rating")
@@ -92,7 +92,7 @@ func TestUDLPurgeUser_LeavesOtherAccountsIntact(t *testing.T) {
 // The key layout separates them with '/' and ':', so this passes only as long
 // as the scan patterns keep those separators.
 func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "cidOne", "like")
 	seedCell(t, c, "zAliceExtra", "cidOne", "like")
@@ -114,7 +114,7 @@ func TestUDLPurgeUser_DoesNotMatchUIDsByPrefix(t *testing.T) {
 // index instead of the cells would leave every retracted rating behind — the
 // rows most likely to have accumulated over an account's life.
 func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "cidOne", "like")
 	if ok, err := c.UDLUpsertIfNewer("zAlice", "cidOne", "like", 2, 200, "rec-tomb", "", false, true); err != nil || !ok {
@@ -140,7 +140,7 @@ func TestUDLPurgeUser_RemovesTombstonedCellsToo(t *testing.T) {
 // Retry-safe: the delete handler purges before removing the key, so a caller
 // that retries after a partial failure runs this a second time.
 func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 	seedCell(t, c, "zAlice", "cidOne", "like")
 
 	if _, err := c.UDLPurgeUser("zAlice"); err != nil {
@@ -156,7 +156,7 @@ func TestUDLPurgeUser_IsIdempotent(t *testing.T) {
 }
 
 func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 	seedCell(t, c, "zAlice", "cidOne", "like")
 
 	// "*" would otherwise match every account in the keyspace — a wildcard
@@ -172,7 +172,7 @@ func TestUDLPurgeUser_RefusesGlobInjection(t *testing.T) {
 }
 
 func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "cidOne", "like")
 	seedCell(t, c, "zAlice", "cidOne", "rating")
@@ -198,7 +198,7 @@ func TestUDLAllUserStats_CountsEachAccountSeparately(t *testing.T) {
 // The stats and the purge must agree: the number shown in the delete
 // confirmation is the number of rows the delete then removes.
 func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	seedCell(t, c, "zAlice", "cidOne", "like")
 	seedCell(t, c, "zAlice", "cidTwo", "rating")
@@ -217,26 +217,6 @@ func TestUDLAllUserStats_AgreesWithWhatPurgeRemoves(t *testing.T) {
 	}
 }
 
-// The key prefix (multi-tenant Redis) must not leak into the parsed uid, or
-// every account on a prefixed node would be reported under a mangled name and
-// the purge would find nothing.
-func TestUDLAdmin_RespectsKeyPrefix(t *testing.T) {
-	c, _ := newTestClient(t, "tenantA:")
-
-	seedCell(t, c, "zAlice", "cidOne", "like")
-
-	stats, err := c.UDLAllUserStats()
-	if err != nil {
-		t.Fatalf("stats: %v", err)
-	}
-	if got := stats["zAlice"]; got.Records != 1 {
-		t.Fatalf("stats under a prefix = %+v, want 1 record for zAlice", stats)
-	}
-	if s, err := c.UDLPurgeUser("zAlice"); err != nil || s.Records != 1 {
-		t.Fatalf("purge under a prefix: %+v err=%v", s, err)
-	}
-}
-
 func TestSplitUDLRecSuffix_KeepsSlashesInTheKey(t *testing.T) {
 	// A namespaced key arriving later must not be truncated: the purge SREMs
 	// `udl:idx:cid:<cid>:key:<key>`, so a half-read key removes the wrong

internal/storage/udl_export_test.go

@@ -0,0 +1,160 @@
+package storage
+
+import (
+	"testing"
+)
+
+// The whole-profile export is what cross-instance recovery reads: a box that
+// has just imported a secret key knows the uid and nothing else — not which
+// cids the profile touched, not which keys. Every one of these tests is about
+// that one caller.
+
+func TestUDLAllForUser_ReturnsEveryCellAcrossCidsAndKeys(t *testing.T) {
+	c := newTestClient(t)
+
+	const uid = "zAlice"
+	writes := []struct{ cid, key, rec string }{
+		{"bagcsaaaaone", "like", "rec-one-like"},
+		{"bagcsaaaaone", "seek", "rec-one-seek"},
+		{"bagcsaaaatwo", "rating", "rec-two-rating"},
+		{"bagcsaaaatwo", "profile:name", "rec-two-name"},
+		{"bagcsaaathree", "watched", "rec-three-watched"},
+	}
+	for _, w := range writes {
+		if ok, err := c.UDLUpsertIfNewer(uid, w.cid, w.key, 1, 100, w.rec, "", false, false); err != nil || !ok {
+			t.Fatalf("upsert %s/%s: ok=%v err=%v", w.cid, w.key, ok, err)
+		}
+	}
+
+	entries, next, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if next != "" {
+		t.Fatalf("export of 5 rows should be complete, got nextCid=%q", next)
+	}
+	if len(entries) != len(writes) {
+		t.Fatalf("want %d rows, got %d: %+v", len(writes), len(entries), entries)
+	}
+	seen := map[string]string{}
+	for _, e := range entries {
+		seen[e.Cid+"/"+e.Key] = e.Record
+	}
+	for _, w := range writes {
+		if got := seen[w.cid+"/"+w.key]; got != w.rec {
+			t.Errorf("%s/%s: want record %q, got %q", w.cid, w.key, w.rec, got)
+		}
+	}
+}
+
+// The failure this guards is silent and total: a uid whose cids were never
+// indexed exports as empty, and recovery reports success having restored
+// nothing.
+func TestUDLAllForUser_IsEmptyForAnUnknownUid(t *testing.T) {
+	c := newTestClient(t)
+
+	if ok, err := c.UDLUpsertIfNewer("zAlice", "bagcsaaaaone", "like", 1, 100, "rec", "", false, false); err != nil || !ok {
+		t.Fatalf("upsert: ok=%v err=%v", ok, err)
+	}
+	entries, _, err := c.UDLAllForUser("zBob", "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if len(entries) != 0 {
+		t.Fatalf("Bob owns nothing; got %+v", entries)
+	}
+}
+
+// ⚠ The tombstone rule, and the reason the cids index is never pruned.
+//
+// A tombstone must EXPORT. It is the only thing that can beat the older live
+// value the other box still holds at a lower version — drop it from the export
+// and an un-liked title comes back from the dead on every peer that already
+// saw the `like`. This is the one place the cids index deliberately diverges
+// from the user+key index, which does prune.
+func TestUDLAllForUser_ExportsTombstones(t *testing.T) {
+	c := newTestClient(t)
+
+	const (
+		uid = "zAlice"
+		cid = "bagcsaaaaone"
+		key = "like"
+	)
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-liked", "", false, false); err != nil || !ok {
+		t.Fatalf("live upsert: ok=%v err=%v", ok, err)
+	}
+	if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 2, 200, "rec-unliked", "", false, true); err != nil || !ok {
+		t.Fatalf("tombstone upsert: ok=%v err=%v", ok, err)
+	}
+
+	// The user+key index prunes it — that is what keeps My List short.
+	live, err := c.UDLListUserKey(uid, key)
+	if err != nil {
+		t.Fatalf("active_for_user_key: %v", err)
+	}
+	if len(live) != 0 {
+		t.Fatalf("tombstoned cid should leave the user+key index, got %+v", live)
+	}
+
+	// The export must still carry it, at the tombstone's version.
+	entries, _, err := c.UDLAllForUser(uid, "", 0)
+	if err != nil {
+		t.Fatalf("export: %v", err)
+	}
+	if len(entries) != 1 {
+		t.Fatalf("tombstone must export, got %+v", entries)
+	}
+	if entries[0].Record != "rec-unliked" || entries[0].Version != 2 {
+		t.Fatalf("export must carry the tombstone, not the superseded value: %+v", entries[0])
+	}
+}
+
+func TestUDLAllForUser_PagesOnACidBoundaryAndResumes(t *testing.T) {
+	c := newTestClient(t)
+
+	const uid = "zAlice"
+	cids := []string{"bagcsaaaaa1", "bagcsaaaaa2", "bagcsaaaaa3"}
+	for _, cid := range cids {
+		for _, key := range []string{"like", "seek"} {
+			if ok, err := c.UDLUpsertIfNewer(uid, cid, key, 1, 100, "rec-"+cid+"-"+key, "", false, false); err != nil || !ok {
+				t.Fatalf("upsert: ok=%v err=%v", ok, err)
+			}
+		}
+	}
+
+	// A cid's keys are never split across a page, so a limit of 1 still yields
+	// both of the first cid's rows.
+	page1, next1, err := c.UDLAllForUser(uid, "", 1)
+	if err != nil {
+		t.Fatalf("page 1: %v", err)
+	}
+	if len(page1) != 2 || page1[0].Cid != cids[0] {
+		t.Fatalf("page 1 should hold both rows of %s, got %+v", cids[0], page1)
+	}
+	if next1 != cids[0] {
+		t.Fatalf("cursor should name the last cid emitted, want %q got %q", cids[0], next1)
+	}
+
+	// Walking the cursor must visit every row exactly once.
+	all := append([]UDLUserAllEntry{}, page1...)
+	cursor := next1
+	for cursor != "" {
+		page, next, err := c.UDLAllForUser(uid, cursor, 1)
+		if err != nil {
+			t.Fatalf("page after %q: %v", cursor, err)
+		}
+		all = append(all, page...)
+		cursor = next
+	}
+	if len(all) != 6 {
+		t.Fatalf("want 6 rows across all pages, got %d: %+v", len(all), all)
+	}
+	seen := map[string]bool{}
+	for _, e := range all {
+		k := e.Cid + "/" + e.Key
+		if seen[k] {
+			t.Fatalf("%s emitted twice across pages", k)
+		}
+		seen[k] = true
+	}
+}

internal/storage/udl_test.go

@@ -15,7 +15,7 @@ import (
 // forever. The reader filters them out again, so the cost is invisible until
 // the account is old.
 func TestUDLUpsert_TombstonePrunesUserKeyIndexButKeepsCell(t *testing.T) {
-	c, _ := newTestClient(t, "")
+	c := newTestClient(t)
 
 	const (
 		uid = "zAlice"

internal/storage/uuid.go

@@ -8,7 +8,7 @@ import (
 // layout). Used as the root key for file metadata entries.
 //
 // Why this format over canonical 8-4-4-4-12 hex:
-//   - 26 chars vs 36 — shorter Redis keys, less prefix space waste
+//   - 26 chars vs 36 — shorter keys, less prefix space waste
 //   - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
 //     given time window, restoring some of the debug-ability lost vs the
 //     previous midhash256:abc rooting

internal/storage/write_bench_test.go

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+)
+
+// BenchmarkConcurrentRecordWrites is the ingest shape the single-writer model
+// has to absorb: many goroutines (feeder flushes, gateway persists, plugin
+// callbacks) each writing a ~40-field record. SQLite serialises them; this
+// measures what that costs. Reported as records/s.
+func BenchmarkConcurrentRecordWrites(b *testing.B) {
+	c := NewClient()
+	if err := c.Connect(filepath.Join(b.TempDir(), "bench.db")); err != nil {
+		b.Fatal(err)
+	}
+	defer c.Close()
+
+	record := func(i int64) map[string]string {
+		m := map[string]string{
+			"title":                               fmt.Sprintf("Some Show S01E%02d", i%40),
+			"fileName":                            fmt.Sprintf("some.show.s01e%02d.1080p.mkv", i%40),
+			"filePath":                            fmt.Sprintf("/files/watch/some.show.%d.mkv", i),
+			"fileType":                            "video",
+			"tmdbid":                              fmt.Sprintf("%d", 1000+i%500),
+			fmt.Sprintf("cids/bafkbench%020d", i): "true",
+		}
+		for f := 0; len(m) < 40; f++ {
+			m[fmt.Sprintf("categories/%d", f)] = "true"
+		}
+		return m
+	}
+
+	const writers = 16
+	var next int64
+	b.ResetTimer()
+	var wg sync.WaitGroup
+	for w := 0; w < writers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				i := atomic.AddInt64(&next, 1)
+				if i > int64(b.N) {
+					return
+				}
+				if err := c.SetMetadataFlat(fmt.Sprintf("01BENCH%019d", i), record(i)); err != nil {
+					b.Error(err)
+					return
+				}
+			}
+		}()
+	}
+	wg.Wait()
+	b.ReportMetric(float64(b.N)/b.Elapsed().Seconds(), "records/s")
+}

internal/watcher/dispatcher.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	// EventsStream is the Redis Stream name for file events
+	// EventsStream is the stream name for file events
 	EventsStream = "file:events"
 
 	// EventsStreamMaxLen is the approximate cap on stream length. Bounded
@@ -20,7 +20,7 @@ const (
 	EventsStreamMaxLen = 100_000
 )
 
-// Dispatcher sends file events to Redis Stream
+// Dispatcher sends file events to the file:events stream
 type Dispatcher struct {
 	mu            sync.RWMutex
 	storageClient *storage.Client
@@ -31,20 +31,20 @@ func NewDispatcher() *Dispatcher {
 	return &Dispatcher{}
 }
 
-// SetStorageClient sets the storage client for Redis Stream publishing
+// SetStorageClient sets the storage client for file:events publishing
 func (d *Dispatcher) SetStorageClient(client *storage.Client) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	d.storageClient = client
-	log.Println("[Dispatcher] Storage client set for Redis Stream publishing")
+	log.Println("[Dispatcher] Storage client set for file:events publishing")
 }
 
-// Dispatch sends an event to Redis Stream
+// Dispatch sends an event to the file:events stream
 func (d *Dispatcher) Dispatch(event FileEvent) {
 	go d.dispatchToStream(event)
 }
 
-// dispatchToStream publishes event to Redis Stream for service consumption
+// dispatchToStream publishes event to the file:events stream for service consumption
 func (d *Dispatcher) dispatchToStream(event FileEvent) {
 	d.mu.RLock()
 	client := d.storageClient
@@ -125,7 +125,7 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
 	return nil
 }
 
-// GetRecentEvents reads recent events from the Redis stream, newest first.
+// GetRecentEvents reads recent events from the stream, newest first.
 // If sinceMS > 0, only events strictly newer than that timestamp are returned.
 // limit caps the number of returned events (defaults to 100 when <= 0).
 func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
@@ -160,7 +160,7 @@ func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, err
 	return events, nil
 }
 
-// parseStreamEntry converts a Redis stream entry back into a FileEvent.
+// parseStreamEntry converts a stream entry back into a FileEvent.
 // All fields are stored as strings by XADD, so numerics need parsing.
 func parseStreamEntry(entry storage.StreamEntry) FileEvent {
 	event := FileEvent{}
@@ -192,7 +192,7 @@ func parseStreamEntry(entry storage.StreamEntry) FileEvent {
 	}
 
 	// Fall back to the stream ID's millisecond prefix if the timestamp
-	// field is missing or unparseable — every Redis stream ID starts with ms.
+	// field is missing or unparseable — every stream ID starts with ms.
 	if event.Timestamp == 0 && entry.ID != "" {
 		if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
 			if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {

internal/watcher/handlers.go

@@ -25,7 +25,7 @@ func NewHandlers(watcher *Watcher, dispatcher *Dispatcher) *Handlers {
 
 // RegisterRoutes registers all watcher-related routes
 func (h *Handlers) RegisterRoutes(r *mux.Router) {
-	// Event polling (for services that don't use Redis Streams)
+	// Event polling (for services that don't consume the SSE stream)
 	r.HandleFunc("/api/events/poll", h.handlePoll).Methods("GET")
 
 	// Deprecated scan endpoints - redirect to /api/watchers
@@ -61,7 +61,7 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// Read events from the Redis `file:events` stream rather than the
+	// Read events from the `file:events` stream rather than the
 	// in-memory buffer. The buffer is fed by the unused debouncer path,
 	// so it is always empty in practice — the stream is the source of truth.
 	events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)

internal/watcher/watcher.go

@@ -11,7 +11,6 @@ import (
 	"github.com/metazla/meta-core/internal/storage"
 )
 
-
 // Watcher provides file scanning and event dispatching
 type Watcher struct {
 	config        *config.Config
@@ -26,8 +25,8 @@ type Watcher struct {
 }
 
 // SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
-// tuple to Redis whenever it computes a midhash. Without this, only the
-// in-memory state registry knows about the file — Redis would only learn
+// tuple to the store whenever it computes a midhash. Without this, only the
+// in-memory state registry knows about the file — the store would only learn
 // about it via downstream consumers of the event stream.
 func (w *Watcher) SetStorageClient(s *storage.Client) {
 	w.storage = s
@@ -35,7 +34,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
 
 // writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
 // Caller already has all four values from its existing os.Stat / FileInfo —
-// no extra IO is performed here beyond the Redis writes themselves.
+// no extra IO is performed here beyond the store writes themselves.
 //
 // Flow:
 //   - Look up the midhash CID in the reverse index (cid:<midhash> → uuid)
@@ -46,7 +45,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
 //     record the new path in the duplicates set and refresh size/mtime
 //
 // This is also the inverse-index that lets a fresh meta-core boot skip
-// re-mid-hashing files whose (path, size, mtime) match what's in Redis,
+// re-mid-hashing files whose (path, size, mtime) match what's stored,
 // via HydrateStateFromStorage below.
 func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
 	if w.storage == nil || midhash == "" {
@@ -270,7 +269,7 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
 }
 
 // WatcherRoot pairs a watcher's id with the absolute path it watches. The
-// hydrator uses Root to decide which Redis-tracked file belongs to which
+// hydrator uses Root to decide which stored file belongs to which
 // in-memory WatcherState.
 type WatcherRoot struct {
 	ID   string
@@ -278,7 +277,7 @@ type WatcherRoot struct {
 }
 
 // HydrateStateFromStorage repopulates the per-watcher in-memory state from
-// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
+// the (filePath, sizeByte, mtimeNano) tuples in the store. After this runs, the
 // next scan tick will see existing files as "known" and skip
 // ComputeMidHash256 for any whose size+mtime match — turning a cold-start
 // rescan from minutes to seconds on a populated library.
@@ -416,4 +415,3 @@ func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
 func (w *Watcher) GetFilesPath() string {
 	return w.filesPath
 }
-

internal/watchers/poller.go

@@ -74,7 +74,7 @@ func (p *Poller) Start() error {
 	// consumer state without buying anything. The historical reason for
 	// wiping — "in-memory state is empty, all files re-emit as add" —
 	// stopped being true when HydrateStateFromStorage landed: state is
-	// hydrated from Redis before this runs, so a startup EmitReset would
+	// hydrated from the store before this runs, so a startup EmitReset would
 	// tell consumers to throw out their world without sending the
 	// re-emission they need to rebuild it.
 	//
@@ -308,7 +308,7 @@ func (p *Poller) TriggerScanAll() (int, error) {
 }
 
 // TriggerReset performs a full reset for a watcher:
-// 1. Clears the Redis stream and emits a reset event
+// 1. Clears the file:events stream and emits a reset event
 // 2. Clears the watcher's state
 // 3. Performs a full scan (all files are "new")
 func (p *Poller) TriggerReset(watcherID string) (int, error) {

test/config_test.go

@@ -30,8 +30,8 @@ func TestConfigDefaults(t *testing.T) {
 		t.Errorf("Expected HTTPPort 9000, got %d", cfg.HTTPPort)
 	}
 
-	if cfg.RedisPort != 6379 {
-		t.Errorf("Expected RedisPort 6379, got %d", cfg.RedisPort)
+	if cfg.DBPath() != cfg.MetaCorePath+"/db/meta-core.db" {
+		t.Errorf("Expected DBPath under MetaCorePath, got %q", cfg.DBPath())
 	}
 }
 
@@ -41,7 +41,7 @@ func TestConfigFromEnv(t *testing.T) {
 	os.Setenv("FILES_PATH", "/custom/files")
 	os.Setenv("SERVICE_NAME", "test-service")
 	os.Setenv("META_CORE_HTTP_PORT", "8080")
-	os.Setenv("REDIS_PORT", "6380")
+	os.Setenv("META_CORE_DB_PATH", "/custom/db.sqlite")
 	defer os.Clearenv()
 
 	cfg := config.Load()
@@ -62,8 +62,8 @@ func TestConfigFromEnv(t *testing.T) {
 		t.Errorf("Expected HTTPPort 8080, got %d", cfg.HTTPPort)
 	}
 
-	if cfg.RedisPort != 6380 {
-		t.Errorf("Expected RedisPort 6380, got %d", cfg.RedisPort)
+	if cfg.DBPath() != "/custom/db.sqlite" {
+		t.Errorf("Expected DBPath '/custom/db.sqlite', got %q", cfg.DBPath())
 	}
 }
 
@@ -77,15 +77,8 @@ func TestConfigPaths(t *testing.T) {
 		t.Errorf("Unexpected LockFilePath: %s", cfg.LockFilePath())
 	}
 
-	if cfg.InfoFilePath() != "/test/locks/kv-leader.info" {
-		t.Errorf("Unexpected InfoFilePath: %s", cfg.InfoFilePath())
+	if cfg.DBPath() != "/test/db/meta-core.db" {
+		t.Errorf("Unexpected DBPath: %s", cfg.DBPath())
 	}
 
-	if cfg.RedisDataDir() != "/test/db/redis" {
-		t.Errorf("Unexpected RedisDataDir: %s", cfg.RedisDataDir())
-	}
-
-	if cfg.ServicesDir() != "/test/services" {
-		t.Errorf("Unexpected ServicesDir: %s", cfg.ServicesDir())
-	}
 }

test/election_test.go

@@ -1,144 +0,0 @@
-package test
-
-import (
-	"encoding/json"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/metazla/meta-core/internal/leader"
-)
-
-func TestLeaderLockInfoJSON(t *testing.T) {
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		BaseURL:   "http://localhost:8180",
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	// Marshal to JSON
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	// Verify JSON structure matches TypeScript format
-	var parsed map[string]interface{}
-	if err := json.Unmarshal(data, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if parsed["host"] != "test-host" {
-		t.Errorf("Expected host 'test-host', got '%v'", parsed["host"])
-	}
-
-	if parsed["api"] != "redis://10.0.0.1:6379" {
-		t.Errorf("Expected api 'redis://10.0.0.1:6379', got '%v'", parsed["api"])
-	}
-
-	if parsed["http"] != "http://10.0.0.1:8180" {
-		t.Errorf("Expected http 'http://10.0.0.1:8180', got '%v'", parsed["http"])
-	}
-
-	if parsed["baseUrl"] != "http://localhost:8180" {
-		t.Errorf("Expected baseUrl 'http://localhost:8180', got '%v'", parsed["baseUrl"])
-	}
-
-	// Timestamp should be a number
-	if _, ok := parsed["timestamp"].(float64); !ok {
-		t.Errorf("Expected timestamp to be a number, got %T", parsed["timestamp"])
-	}
-
-	// PID should be a number
-	if _, ok := parsed["pid"].(float64); !ok {
-		t.Errorf("Expected pid to be a number, got %T", parsed["pid"])
-	}
-}
-
-func TestLeaderLockInfoJSONOmitEmpty(t *testing.T) {
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		BaseURL:   "", // Empty - should be omitted
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	data, err := json.Marshal(info)
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	// baseUrl should not be present when empty
-	var parsed map[string]interface{}
-	if err := json.Unmarshal(data, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if _, exists := parsed["baseUrl"]; exists {
-		t.Error("Expected baseUrl to be omitted when empty")
-	}
-}
-
-func TestLeaderInfoFileFormat(t *testing.T) {
-	// Create a temp directory
-	tmpDir, err := os.MkdirTemp("", "meta-core-test")
-	if err != nil {
-		t.Fatalf("Failed to create temp dir: %v", err)
-	}
-	defer os.RemoveAll(tmpDir)
-
-	info := &leader.LeaderLockInfo{
-		Host:      "test-host",
-		API:       "redis://10.0.0.1:6379",
-		HTTP:      "http://10.0.0.1:8180",
-		Timestamp: 1704067200000,
-		PID:       12345,
-	}
-
-	// Write to file
-	infoPath := filepath.Join(tmpDir, "kv-leader.info")
-	data, err := json.MarshalIndent(info, "", "  ")
-	if err != nil {
-		t.Fatalf("Failed to marshal: %v", err)
-	}
-
-	if err := os.WriteFile(infoPath, data, 0644); err != nil {
-		t.Fatalf("Failed to write file: %v", err)
-	}
-
-	// Read and parse
-	readData, err := os.ReadFile(infoPath)
-	if err != nil {
-		t.Fatalf("Failed to read file: %v", err)
-	}
-
-	var parsed leader.LeaderLockInfo
-	if err := json.Unmarshal(readData, &parsed); err != nil {
-		t.Fatalf("Failed to unmarshal: %v", err)
-	}
-
-	if parsed.Host != info.Host {
-		t.Errorf("Host mismatch: expected '%s', got '%s'", info.Host, parsed.Host)
-	}
-
-	if parsed.API != info.API {
-		t.Errorf("API mismatch: expected '%s', got '%s'", info.API, parsed.API)
-	}
-
-	if parsed.HTTP != info.HTTP {
-		t.Errorf("HTTP mismatch: expected '%s', got '%s'", info.HTTP, parsed.HTTP)
-	}
-
-	if parsed.Timestamp != info.Timestamp {
-		t.Errorf("Timestamp mismatch: expected %d, got %d", info.Timestamp, parsed.Timestamp)
-	}
-
-	if parsed.PID != info.PID {
-		t.Errorf("PID mismatch: expected %d, got %d", info.PID, parsed.PID)
-	}
-}